{"id":"https://openalex.org/W4415318663","doi":"https://doi.org/10.48550/arxiv.2510.08240","title":"The Alignment Waltz: Jointly Training Agents to Collaborate for Safety","display_name":"The Alignment Waltz: Jointly Training Agents to Collaborate for Safety","publication_year":2025,"publication_date":"2025-10-09","ids":{"openalex":"https://openalex.org/W4415318663","doi":"https://doi.org/10.48550/arxiv.2510.08240"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2510.08240","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2510.08240","pdf_url":"https://arxiv.org/pdf/2510.08240","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2510.08240","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100335725","display_name":"Jingyu Zhang","orcid":"https://orcid.org/0009-0000-9142-990X","observed_orcids":["https://orcid.org/0009-0000-9142-990X","https://orcid.org/0000-0001-9901-8611"]},"institutions":[{"id":"https://openalex.org/I145311948","display_name":"Johns Hopkins University","ror":"https://ror.org/00za53h95","country_code":"US","type":"education","lineage":["https://openalex.org/I145311948"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Zhang, Jingyu","raw_affiliation_strings":["Johns Hopkins University","Meta Superintelligence Labs ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Johns Hopkins University","institution_ids":["https://openalex.org/I145311948"]},{"raw_affiliation_string":"Meta Superintelligence Labs ,","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5076722448","display_name":"Haozhu Wang","orcid":"https://orcid.org/0000-0002-9679-0144","observed_orcids":["https://orcid.org/0000-0002-9679-0144"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Haozhu","raw_affiliation_strings":["Meta Superintelligence Labs ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Meta Superintelligence Labs ,","institution_ids":[]}]},{"author_position":"middle","author":{"id":null,"display_name":"Smith, Eric Michael","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Smith, Eric Michael","raw_affiliation_strings":["Meta Superintelligence Labs ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Meta Superintelligence Labs ,","institution_ids":[]}]},{"author_position":"middle","author":{"id":null,"display_name":"Wang, Sid","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Sid","raw_affiliation_strings":["Meta Superintelligence Labs ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Meta Superintelligence Labs ,","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5068614789","display_name":"Amr Sharaf","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sharaf, Amr","raw_affiliation_strings":["Meta Superintelligence Labs ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Meta Superintelligence Labs ,","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5010699741","display_name":"Mahesh Babu Pasupuleti","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Pasupuleti, Mahesh","raw_affiliation_strings":["Meta Superintelligence Labs ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Meta Superintelligence Labs ,","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5075825791","display_name":"Benjamin Van Durme","orcid":"https://orcid.org/0000-0003-4328-4288","observed_orcids":["https://orcid.org/0000-0003-4328-4288"]},"institutions":[{"id":"https://openalex.org/I145311948","display_name":"Johns Hopkins University","ror":"https://ror.org/00za53h95","country_code":"US","type":"education","lineage":["https://openalex.org/I145311948"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Van Durme, Benjamin","raw_affiliation_strings":["Johns Hopkins University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Johns Hopkins University","institution_ids":["https://openalex.org/I145311948"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5043628255","display_name":"Daniel Khashabi","orcid":"https://orcid.org/0009-0009-7664-2230","observed_orcids":["https://orcid.org/0009-0009-7664-2230"]},"institutions":[{"id":"https://openalex.org/I145311948","display_name":"Johns Hopkins University","ror":"https://ror.org/00za53h95","country_code":"US","type":"education","lineage":["https://openalex.org/I145311948"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Khashabi, Daniel","raw_affiliation_strings":["Johns Hopkins University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Johns Hopkins University","institution_ids":["https://openalex.org/I145311948"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5076635608","display_name":"Jason Weston","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Weston, Jason","raw_affiliation_strings":["Meta Superintelligence Labs ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Meta Superintelligence Labs ,","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5064285315","display_name":"Hongyuan Zhan","orcid":"https://orcid.org/0000-0002-4130-0451","observed_orcids":["https://orcid.org/0000-0002-4130-0451"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhan, Hongyuan","raw_affiliation_strings":["Meta Superintelligence Labs ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Meta Superintelligence Labs ,","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":5,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9779999852180481,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9779999852180481,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.0052999998442828655,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12262","display_name":"Hate Speech and Cyberbullying Detection","score":0.004800000227987766,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/large-language-model-safety","display_name":"large language model safety","score":0.7609999775886536},{"id":"https://openalex.org/keywords/safety-alignment","display_name":"safety alignment","score":0.800000011920929},{"id":"https://openalex.org/keywords/multi-agent-reinforcement-learning","display_name":"multi-agent reinforcement learning","score":0.9190000295639038},{"id":"https://openalex.org/keywords/adversarial-attacks","display_name":"adversarial attacks","score":0.6439999938011169},{"id":"https://openalex.org/keywords/joint-training","display_name":"joint training","score":0.3370000123977661},{"id":"https://openalex.org/keywords/adaptive-feedback","display_name":"adaptive feedback","score":0.46299999952316284}],"concepts":[{"id":"https://openalex.org/C2777200299","wikidata":"https://www.wikidata.org/wiki/Q52943","display_name":"Conversation","level":2,"score":0.9032999873161316},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6687999963760376},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.486299991607666},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.4668000042438507},{"id":"https://openalex.org/C194995250","wikidata":"https://www.wikidata.org/wiki/Q531136","display_name":"Affordance","level":2,"score":0.4596000015735626},{"id":"https://openalex.org/C2781265381","wikidata":"https://www.wikidata.org/wiki/Q5710255","display_name":"Helpfulness","level":2,"score":0.4339999854564667},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.4025999903678894},{"id":"https://openalex.org/C95713431","wikidata":"https://www.wikidata.org/wiki/Q631425","display_name":"Vulnerability (computing)","level":2,"score":0.36329999566078186},{"id":"https://openalex.org/C147446459","wikidata":"https://www.wikidata.org/wiki/Q11639","display_name":"Dance","level":2,"score":0.3273000121116638},{"id":"https://openalex.org/C112930515","wikidata":"https://www.wikidata.org/wiki/Q4389547","display_name":"Risk analysis (engineering)","level":1,"score":0.31540000438690186},{"id":"https://openalex.org/C2780791683","wikidata":"https://www.wikidata.org/wiki/Q846785","display_name":"Action (physics)","level":2,"score":0.3073999881744385},{"id":"https://openalex.org/C2776654903","wikidata":"https://www.wikidata.org/wiki/Q2601463","display_name":"SAFER","level":2,"score":0.3059999942779541},{"id":"https://openalex.org/C2780522230","wikidata":"https://www.wikidata.org/wiki/Q1140419","display_name":"Ambiguity","level":2,"score":0.3018999993801117},{"id":"https://openalex.org/C2775924081","wikidata":"https://www.wikidata.org/wiki/Q55608371","display_name":"Control (management)","level":2,"score":0.29809999465942383},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.28349998593330383},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2824000120162964},{"id":"https://openalex.org/C108827166","wikidata":"https://www.wikidata.org/wiki/Q175975","display_name":"Internet privacy","level":1,"score":0.2750999927520752},{"id":"https://openalex.org/C163258240","wikidata":"https://www.wikidata.org/wiki/Q25342","display_name":"Power (physics)","level":2,"score":0.27129998803138733},{"id":"https://openalex.org/C139002025","wikidata":"https://www.wikidata.org/wiki/Q3001212","display_name":"Lift (data mining)","level":2,"score":0.259799987077713},{"id":"https://openalex.org/C6683253","wikidata":"https://www.wikidata.org/wiki/Q7075535","display_name":"Obstacle avoidance","level":4,"score":0.25920000672340393}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2510.08240","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2510.08240","pdf_url":"https://arxiv.org/pdf/2510.08240","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2510.08240","doi":"https://doi.org/10.48550/arxiv.2510.08240","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2510.08240","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2510.08240","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2510.08240","pdf_url":"https://arxiv.org/pdf/2510.08240","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4415318663.pdf","grobid_xml":"https://content.openalex.org/works/W4415318663.grobid-xml"},"referenced_works_count":36,"referenced_works":["https://openalex.org/W2736601468","https://openalex.org/W3201174429","https://openalex.org/W4223908421","https://openalex.org/W4382318449","https://openalex.org/W4385894687","https://openalex.org/W4387321091","https://openalex.org/W4388718089","https://openalex.org/W4389072543","https://openalex.org/W4389713766","https://openalex.org/W4391631327","https://openalex.org/W4392539041","https://openalex.org/W4393946427","https://openalex.org/W4394708144","https://openalex.org/W4399317891","https://openalex.org/W4399794742","https://openalex.org/W4401042900","https://openalex.org/W4401043746","https://openalex.org/W4402670694","https://openalex.org/W4402671666","https://openalex.org/W4403443810","https://openalex.org/W4404344587","https://openalex.org/W4404782684","https://openalex.org/W4405273924","https://openalex.org/W4407093355","https://openalex.org/W4412505606","https://openalex.org/W4414903627","https://openalex.org/W4414987658","https://openalex.org/W4415068033","https://openalex.org/W4415109167","https://openalex.org/W4415327992","https://openalex.org/W4415799204","https://openalex.org/W4416097298","https://openalex.org/W4416169877","https://openalex.org/W6884962196","https://openalex.org/W6902634636","https://openalex.org/W6959934489"],"related_works":["https://openalex.org/W7125781240","https://openalex.org/W7127451373","https://openalex.org/W7220915565","https://openalex.org/W4417275640","https://openalex.org/W7126001943","https://openalex.org/W4387838992","https://openalex.org/W7220466991","https://openalex.org/W4407719102","https://openalex.org/W7123336998","https://openalex.org/W7166835435"],"abstract_inverted_index":{"Harnessing":[0],"the":[1,62,114,124,129,134,152,156,166,181,234,255],"power":[2],"of":[3,128,136],"LLMs":[4],"requires":[5],"a":[6,16,32,81,88,99,106,110,139],"delicate":[7],"dance":[8,46],"between":[9,19,258],"being":[10],"helpful":[11],"and":[12,31,68,109,126,184,192,220,236,241,260],"harmless.":[13],"This":[14,59],"creates":[15],"fundamental":[17],"tension":[18],"two":[20],"competing":[21],"challenges:":[22],"vulnerability":[23],"to":[24,70,118,216,224,229,239],"adversarial":[25],"attacks":[26],"that":[27,50,55,94,122,144,206],"elicit":[28],"unsafe":[29,57,161,211],"content,":[30],"tendency":[33],"for":[34,74],"overrefusal":[35],"on":[36,149,195,218,226],"benign":[37],"but":[38],"sensitive":[39],"prompts.":[40],"Current":[41],"approaches":[42],"often":[43],"navigate":[44],"this":[45],"with":[47,180],"safeguard":[48],"models":[49,80],"completely":[51],"reject":[52],"any":[53],"content":[54],"contains":[56],"portions.":[58],"approach":[60],"cuts":[61],"music":[63],"entirely-it":[64],"may":[65],"exacerbate":[66],"overrefusals":[67,221],"fails":[69],"provide":[71,119],"nuanced":[72],"guidance":[73],"queries":[75],"it":[76],"refuses.":[77],"To":[78],"teach":[79],"more":[82],"coordinated":[83],"choreography,":[84],"we":[85],"propose":[86],"WaltzRL,":[87],"novel":[89],"multi-agent":[90],"reinforcement":[91],"learning":[92],"framework":[93],"formulates":[95],"safety":[96,125,248],"alignment":[97],"as":[98],"collaborative,":[100],"positive-sum":[101],"game.":[102],"WaltzRL":[103,137,207,245],"jointly":[104],"trains":[105],"conversation":[107,130,153,167,182,235],"agent":[108,154,168,176,183],"feedback":[111,175,237],"agent,":[112],"where":[113],"latter":[115],"is":[116,138,177],"incentivized":[117],"useful":[120],"suggestions":[121],"improve":[123],"helpfulness":[127,191,259],"agent's":[131],"responses.":[132],"At":[133,158],"core":[135],"Dynamic":[140],"Improvement":[141],"Reward":[142],"(DIR)":[143],"evolves":[145],"over":[146],"time":[147],"based":[148],"how":[150],"well":[151],"incorporates":[155],"feedback.":[157],"inference":[159],"time,":[160],"or":[162],"overrefusing":[163],"responses":[164,212],"from":[165,214],"are":[169],"improved":[170],"rather":[171],"than":[172],"discarded.":[173],"The":[174],"deployed":[178],"together":[179],"only":[185],"engages":[186],"adaptively":[187,242],"when":[188],"needed,":[189],"preserving":[190],"low":[193],"latency":[194],"safe":[196],"queries.":[197],"Our":[198],"experiments,":[199],"conducted":[200],"across":[201],"five":[202],"diverse":[203],"datasets,":[204],"demonstrate":[205],"significantly":[208],"reduces":[209],"both":[210],"(e.g.,":[213],"39.0%":[215],"4.6%":[217],"WildJailbreak)":[219],"(from":[222],"45.3%":[223],"9.9%":[225],"OR-Bench)":[227],"compared":[228],"various":[230],"baselines.":[231],"By":[232],"enabling":[233],"agents":[238],"co-evolve":[240],"apply":[243],"feedback,":[244],"enhances":[246],"LLM":[247],"without":[249],"degrading":[250],"general":[251],"capabilities,":[252],"thereby":[253],"advancing":[254],"Pareto":[256],"front":[257],"harmlessness.":[261]},"counts_by_year":[{"year":2026,"cited_by_count":4},{"year":2025,"cited_by_count":1}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2025-10-18T00:00:00"}