{"id":"https://openalex.org/W4403709727","doi":"https://doi.org/10.48550/arxiv.2409.11423","title":"Generated Data with Fake Privacy: Hidden Dangers of Fine-tuning Large Language Models on Generated Data","display_name":"Generated Data with Fake Privacy: Hidden Dangers of Fine-tuning Large Language Models on Generated Data","publication_year":2024,"publication_date":"2024-09-12","ids":{"openalex":"https://openalex.org/W4403709727","doi":"https://doi.org/10.48550/arxiv.2409.11423"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2409.11423","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2409.11423","pdf_url":"https://arxiv.org/pdf/2409.11423","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2409.11423","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5139090396","display_name":"Akkus Atilla","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I168864056","display_name":"Bilkent University","ror":"https://ror.org/02vh8a032","country_code":"TR","type":"education","lineage":["https://openalex.org/I168864056"]}],"countries":["TR"],"is_corresponding":false,"raw_author_name":"Akkus, Atilla","raw_affiliation_strings":["Bilkent University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Bilkent University","institution_ids":["https://openalex.org/I168864056"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Aghdam, Masoud Poorghaffar","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I168864056","display_name":"Bilkent University","ror":"https://ror.org/02vh8a032","country_code":"TR","type":"education","lineage":["https://openalex.org/I168864056"]}],"countries":["TR"],"is_corresponding":false,"raw_author_name":"Aghdam, Masoud Poorghaffar","raw_affiliation_strings":["Bilkent University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Bilkent University","institution_ids":["https://openalex.org/I168864056"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100416262","display_name":"Mingjie Li","orcid":"https://orcid.org/0000-0002-6211-3575","observed_orcids":["https://orcid.org/0000-0002-6211-3575"]},"institutions":[{"id":"https://openalex.org/I4210128801","display_name":"Helmholtz Center for Information Security","ror":"https://ror.org/02njgxr09","country_code":"DE","type":"facility","lineage":["https://openalex.org/I1305996414","https://openalex.org/I4210128801"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Li, Mingjie","raw_affiliation_strings":["CISPA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"CISPA","institution_ids":["https://openalex.org/I4210128801"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5106708808","display_name":"Junjie Chu","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210128801","display_name":"Helmholtz Center for Information Security","ror":"https://ror.org/02njgxr09","country_code":"DE","type":"facility","lineage":["https://openalex.org/I1305996414","https://openalex.org/I4210128801"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Chu, Junjie","raw_affiliation_strings":["CISPA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"CISPA","institution_ids":["https://openalex.org/I4210128801"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5004014878","display_name":"Michael Backes A","orcid":"https://orcid.org/0000-0002-9326-6400","observed_orcids":["https://orcid.org/0000-0002-9326-6400","https://orcid.org/0000-0002-7130-9211"]},"institutions":[{"id":"https://openalex.org/I168864056","display_name":"Bilkent University","ror":"https://ror.org/02vh8a032","country_code":"TR","type":"education","lineage":["https://openalex.org/I168864056"]}],"countries":["TR"],"is_corresponding":false,"raw_author_name":"Backes, Michael","raw_affiliation_strings":["Bilkent University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Bilkent University","institution_ids":["https://openalex.org/I168864056"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100354718","display_name":"Yang Zhang","orcid":"https://orcid.org/0000-0002-7632-0460","observed_orcids":["https://orcid.org/0000-0002-7632-0460"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Yang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5025665736","display_name":"Sinem Sav","orcid":"https://orcid.org/0000-0001-9096-8768","observed_orcids":["https://orcid.org/0000-0001-9096-8768"]},"institutions":[{"id":"https://openalex.org/I168864056","display_name":"Bilkent University","ror":"https://ror.org/02vh8a032","country_code":"TR","type":"education","lineage":["https://openalex.org/I168864056"]}],"countries":["TR"],"is_corresponding":false,"raw_author_name":"Sav, Sinem","raw_affiliation_strings":["Bilkent University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Bilkent University","institution_ids":["https://openalex.org/I168864056"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":8,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10764","display_name":"Privacy-Preserving Technologies in Data","score":0.9955999851226807,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10764","display_name":"Privacy-Preserving Technologies in Data","score":0.9955999851226807,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.0020000000949949026,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11045","display_name":"Privacy, Security, and Data Protection","score":0.000699999975040555,"subfield":{"id":"https://openalex.org/subfields/3312","display_name":"Sociology and Political Science"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.7360000014305115},{"id":"https://openalex.org/keywords/synthetic-data-generation","display_name":"synthetic data generation","score":0.7770000100135803},{"id":"https://openalex.org/keywords/privacy-risks","display_name":"privacy risks","score":0.6430000066757202},{"id":"https://openalex.org/keywords/fine-tuning","display_name":"fine-tuning","score":0.7710000276565552},{"id":"https://openalex.org/keywords/membership-inference-attacks","display_name":"membership inference attacks","score":0.9160000085830688},{"id":"https://openalex.org/keywords/supervised-fine-tuning","display_name":"supervised fine-tuning","score":0.9409999847412109},{"id":"https://openalex.org/keywords/pythia","display_name":"Pythia","score":0.8610000014305115},{"id":"https://openalex.org/keywords/data-privacy","display_name":"data privacy","score":0.3140000104904175}],"concepts":[{"id":"https://openalex.org/C108827166","wikidata":"https://www.wikidata.org/wiki/Q175975","display_name":"Internet privacy","level":1,"score":0.6685329079627991},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5754023194313049},{"id":"https://openalex.org/C2779756789","wikidata":"https://www.wikidata.org/wiki/Q28549308","display_name":"Fake news","level":2,"score":0.5438668131828308},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.44920599460601807}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2409.11423","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2409.11423","pdf_url":"https://arxiv.org/pdf/2409.11423","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2409.11423","doi":"https://doi.org/10.48550/arxiv.2409.11423","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2409.11423","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2409.11423","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2409.11423","pdf_url":"https://arxiv.org/pdf/2409.11423","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G4952866186","display_name":null,"funder_award_id":"16KISAO29K","funder_id":"https://openalex.org/F4320321114","funder_display_name":"Bundesministerium f\u00fcr Bildung und Forschung"}],"funders":[{"id":"https://openalex.org/F4320321114","display_name":"Bundesministerium f\u00fcr Bildung und Forschung","ror":"https://ror.org/04pz7b180"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4403709727.pdf","grobid_xml":"https://content.openalex.org/works/W4403709727.grobid-xml"},"referenced_works_count":34,"referenced_works":["https://openalex.org/W2795435272","https://openalex.org/W2951347366","https://openalex.org/W2964121744","https://openalex.org/W3100213383","https://openalex.org/W3118781290","https://openalex.org/W3168867926","https://openalex.org/W3186492090","https://openalex.org/W3207429447","https://openalex.org/W4223908421","https://openalex.org/W4229005866","https://openalex.org/W4288057780","https://openalex.org/W4322718191","https://openalex.org/W4323709074","https://openalex.org/W4362655426","https://openalex.org/W4376167364","https://openalex.org/W4385572634","https://openalex.org/W4385573569","https://openalex.org/W4385573947","https://openalex.org/W4386114274","https://openalex.org/W4387800062","https://openalex.org/W4387985331","https://openalex.org/W4389216598","https://openalex.org/W4390573302","https://openalex.org/W4391801188","https://openalex.org/W4391871574","https://openalex.org/W4392538595","https://openalex.org/W4393140557","https://openalex.org/W4398192691","https://openalex.org/W4399912927","https://openalex.org/W4404987838","https://openalex.org/W4405181291","https://openalex.org/W4406093146","https://openalex.org/W7133233472","https://openalex.org/W7133235460"],"related_works":["https://openalex.org/W4415238118","https://openalex.org/W7163749729","https://openalex.org/W6966979981","https://openalex.org/W4411446757","https://openalex.org/W7221597804","https://openalex.org/W4416010336","https://openalex.org/W4408565710","https://openalex.org/W7164125018","https://openalex.org/W7085778846","https://openalex.org/W4411534947"],"abstract_inverted_index":{"Large":[0],"language":[1],"models":[2,51,77],"(LLMs)":[3],"have":[4],"demonstrated":[5],"significant":[6],"success":[7],"in":[8,256],"various":[9],"domain-specific":[10],"tasks,":[11],"with":[12,22,61,94,102,132,188,241],"their":[13],"performance":[14],"often":[15],"improving":[16],"substantially":[17],"after":[18,186,230],"fine-tuning.":[19],"However,":[20,60],"fine-tuning":[21,101,127,130,187,242],"real-world":[23,58],"data":[24,37,47,71,73,90,104,119,136,146],"introduces":[25,84,109],"privacy":[26,86,107,111,183,207,238,254],"risks.":[27,184],"To":[28],"mitigate":[29],"these":[30,76],"risks,":[31],"developers":[32],"increasingly":[33],"rely":[34],"on":[35,124,171],"synthetic":[36],"generation":[38],"as":[39,46],"an":[40],"alternative":[41],"to":[42,54,91,181],"using":[43,244],"real":[44,70,95],"data,":[45,191,246],"generated":[48,74,89,120,135,190,245],"by":[49,75,113,121,156,201],"traditional":[50],"is":[52,147],"believed":[53],"be":[55],"different":[56],"from":[57],"data.":[59,96],"the":[62,67,115,141,145,172,192,205,213,220,225,236,248],"advanced":[63],"capabilities":[64],"of":[65,118,143,194,209,216,224,253],"LLMs,":[66,122],"distinction":[68],"between":[69],"and":[72,137,166,176],"has":[78],"become":[79],"nearly":[80],"indistinguishable.":[81],"This":[82],"convergence":[83],"similar":[85],"risks":[87,112,239],"for":[88,198,218,250],"those":[92],"associated":[93,240],"Our":[97,233],"study":[98],"investigates":[99],"whether":[100],"LLM-generated":[103],"truly":[105],"enhances":[106],"or":[108],"additional":[110],"examining":[114],"structural":[116],"characteristics":[117],"focusing":[123],"two":[125],"primary":[126],"approaches:":[128],"supervised":[129],"(SFT)":[131],"unstructured":[133,189],"(plain-text)":[134],"self-instruct":[138,231],"tuning.":[139,232],"In":[140],"scenario":[142],"SFT,":[144],"put":[148],"into":[149],"a":[150],"particular":[151],"instruction":[152],"tuning":[153],"format":[154],"used":[155],"previous":[157],"studies.":[158],"We":[159],"use":[160],"Personal":[161],"Information":[162],"Identifier":[163],"(PII)":[164],"leakage":[165],"Membership":[167],"Inference":[168],"Attacks":[169],"(MIAs)":[170],"Pythia":[173,199],"Model":[174],"Suite":[175],"Open":[177],"Pre-trained":[178],"Transformer":[179],"(OPT)":[180],"measure":[182],"Notably,":[185],"rate":[193],"successful":[195],"PII":[196],"extractions":[197],"increased":[200],"over":[202,228],"20%,":[203],"highlighting":[204],"potential":[206,237],"implications":[208],"such":[210,257],"approaches.":[211,258],"Furthermore,":[212],"ROC-AUC":[214],"score":[215],"MIAs":[217],"Pythia-6.9b,":[219],"second":[221],"biggest":[222],"model":[223],"suite,":[226],"increases":[227],"40%":[229],"results":[234],"indicate":[235],"LLMs":[243],"underscoring":[247],"need":[249],"careful":[251],"consideration":[252],"safeguards":[255]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":5}],"updated_date":"2026-11-10T11:39:43.021295","created_date":"2024-10-24T00:00:00"}