{"id":"https://openalex.org/W7204888089","doi":"https://doi.org/10.48550/arxiv.2608.30141","title":"Balancing Privacy, Utility, and Safety in LLM Alignment through Preference Optimization","display_name":"Balancing Privacy, Utility, and Safety in LLM Alignment through Preference Optimization","publication_year":2026,"publication_date":"2026-08-31","ids":{"openalex":"https://openalex.org/W7204888089","doi":"https://doi.org/10.48550/arxiv.2608.30141"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2608.30141","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2608.30141","pdf_url":"https://arxiv.org/pdf/2608.30141","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2608.30141","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5121820398","display_name":"Dishu Yang","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yang, Dishu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5148930083","display_name":"Jingjing Liu","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Jingjing","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5141077484","display_name":"Jize Li","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Jize","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10764","display_name":"Privacy-Preserving Technologies in Data","score":0.9711999893188477,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10764","display_name":"Privacy-Preserving Technologies in Data","score":0.9711999893188477,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.015599999576807022,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.00419999985024333,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/preference-optimization","display_name":"preference optimization","score":0.7689999938011169},{"id":"https://openalex.org/keywords/llm-alignment","display_name":"LLM alignment","score":0.7289999723434448},{"id":"https://openalex.org/keywords/membership-inference-attacks","display_name":"membership inference attacks","score":0.6919999718666077},{"id":"https://openalex.org/keywords/direct-preference-optimization","display_name":"direct preference optimization","score":0.6069999933242798},{"id":"https://openalex.org/keywords/gemma-3","display_name":"Gemma 3","score":0.4970000088214874},{"id":"https://openalex.org/keywords/gemma-2","display_name":"Gemma 2","score":0.7200000286102295},{"id":"https://openalex.org/keywords/4-bit-quantization","display_name":"4-bit quantization","score":0.8069999814033508}],"concepts":[{"id":"https://openalex.org/C2781265381","wikidata":"https://www.wikidata.org/wiki/Q5710255","display_name":"Helpfulness","level":2,"score":0.7038000226020813},{"id":"https://openalex.org/C2781249084","wikidata":"https://www.wikidata.org/wiki/Q908656","display_name":"Preference","level":2,"score":0.6626999974250793},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5291000008583069},{"id":"https://openalex.org/C12725497","wikidata":"https://www.wikidata.org/wiki/Q810247","display_name":"Baseline (sea)","level":2,"score":0.49149999022483826},{"id":"https://openalex.org/C2780148112","wikidata":"https://www.wikidata.org/wiki/Q1432581","display_name":"Proxy (statistics)","level":2,"score":0.4537999927997589},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.4244000017642975},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.4154999852180481},{"id":"https://openalex.org/C2780385302","wikidata":"https://www.wikidata.org/wiki/Q367158","display_name":"Protocol (science)","level":3,"score":0.3978999853134155},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3668000102043152},{"id":"https://openalex.org/C111335779","wikidata":"https://www.wikidata.org/wiki/Q3454686","display_name":"Reduction (mathematics)","level":2,"score":0.3319000005722046},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3025999963283539},{"id":"https://openalex.org/C183115368","wikidata":"https://www.wikidata.org/wiki/Q856577","display_name":"Weighting","level":2,"score":0.29429998993873596},{"id":"https://openalex.org/C137836250","wikidata":"https://www.wikidata.org/wiki/Q984063","display_name":"Optimization problem","level":2,"score":0.28529998660087585},{"id":"https://openalex.org/C2778058735","wikidata":"https://www.wikidata.org/wiki/Q4692253","display_name":"Aggregate data","level":2,"score":0.2824000120162964},{"id":"https://openalex.org/C150921843","wikidata":"https://www.wikidata.org/wiki/Q1170431","display_name":"Resampling","level":2,"score":0.2720000147819519},{"id":"https://openalex.org/C149629883","wikidata":"https://www.wikidata.org/wiki/Q660926","display_name":"Fraction (chemistry)","level":2,"score":0.25130000710487366}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2608.30141","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2608.30141","pdf_url":"https://arxiv.org/pdf/2608.30141","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2608.30141","doi":"https://doi.org/10.48550/arxiv.2608.30141","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2608.30141","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2608.30141","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2608.30141","pdf_url":"https://arxiv.org/pdf/2608.30141","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W7197009782","https://openalex.org/W4400518721","https://openalex.org/W7163053064","https://openalex.org/W7126274551","https://openalex.org/W7127351271","https://openalex.org/W4412888609","https://openalex.org/W4416342251","https://openalex.org/W4404400430","https://openalex.org/W4407806932","https://openalex.org/W7196938767"],"abstract_inverted_index":{"Preference":[0,30,53],"optimization":[1],"is":[2,33,116],"widely":[3],"used":[4],"to":[5,28,137,164,178],"align":[6],"large":[7],"language":[8],"models":[9],"with":[10,35,118,181],"human":[11],"preferences,":[12],"but":[13],"preference-data":[14],"composition":[15],"may":[16],"also":[17],"influence":[18],"privacy-relevant":[19],"memorization.":[20],"We":[21,50,74],"examine":[22],"whether":[23],"adding":[24],"synthetic":[25],"privacy-preference":[26,64,114],"pairs":[27],"Direct":[29],"Optimization":[31],"(DPO)":[32],"associated":[34,117],"lower":[36,119,131],"canary-based":[37],"memorization":[38],"signals":[39],"without":[40],"modifying":[41],"the":[42,61,97,111,138,141,147,151,155,167,171,187,190,203,206],"objective":[43],"or":[44,248],"introducing":[45],"a":[46,56,76,233,244,249],"formal":[47,245],"privacy":[48,207,246],"mechanism.":[49],"propose":[51],"Privacy-Pressure":[52],"Mixing":[54],"(P3M),":[55],"data-composition":[57],"protocol":[58,236],"that":[59,227],"varies":[60,213],"amount":[62],"of":[63,82],"data":[65,72,200],"while":[66],"keeping":[67],"helpfulness":[68,218],"and":[69,79,85,95,130,166,183,209],"harmlessness":[70,210],"preference":[71,211,219],"fixed.":[73],"evaluate":[75],"non-privacy":[77],"Baseline":[78,139],"privacy-mixing":[80],"ratios":[81],"0.5,":[83],"1.0,":[84],"2.0":[86],"using":[87,101],"Gemma":[88,103],"3":[89],"270M-IT":[90],"across":[91,106,126,146,199],"five":[92],"random":[93],"seeds":[94],"validate":[96],"same":[98],"four":[99],"conditions":[100],"4-bit-quantized":[102],"2":[104],"2B-IT":[105],"three":[107],"seeds.":[108],"Overall,":[109],"under":[110,154,170],"tested":[112],"conditions,":[113],"mixing":[115],"mean":[120,152,168],"canary":[121],"suffix":[122],"log-likelihood":[123],"proxy":[124],"values":[125],"both":[127],"model":[128,215],"settings":[129],"aggregate":[132],"membership-inference":[133],"attack":[134],"performance":[135],"relative":[136],"in":[140,192],"mixed-source":[142],"2B":[143,149],"evaluation.":[144],"Specifically,":[145],"privacy-aware":[148],"configurations,":[150],"area":[153,169],"receiver":[156],"operating":[157],"characteristic":[158],"curve":[159,173],"(AUROC)":[160],"ranges":[161,175],"from":[162,176],"0.596":[163],"0.629,":[165],"precision-recall":[172],"(AUPRC)":[174],"0.541":[177],"0.575,":[179],"compared":[180],"0.804":[182],"0.790,":[184],"respectively,":[185],"for":[186,237],"Baseline.":[188],"However,":[189],"reduction":[191],"membership":[193],"distinguishability":[194],"does":[195],"not":[196],"hold":[197],"uniformly":[198],"sources.":[201],"Moreover,":[202],"relationship":[204],"between":[205],"ratio":[208],"accuracy":[212,220],"by":[214],"setting,":[216],"whereas":[217],"remains":[221],"broadly":[222],"stable.":[223],"These":[224],"findings":[225],"suggest":[226],"P3M":[228],"should":[229],"be":[230],"viewed":[231],"as":[232,243],"lightweight":[234],"empirical":[235],"examining":[238],"privacy-utility-safety":[239],"trade-offs":[240],"rather":[241],"than":[242],"guarantee":[247],"defense":[250],"against":[251],"extraction":[252],"attacks.":[253]},"counts_by_year":[],"updated_date":"2026-10-28T10:33:24.094632","created_date":"2026-09-02T00:00:00"}