{"id":"https://openalex.org/W7167594303","doi":"https://doi.org/10.48550/arxiv.2607.02914","title":"Oyster-II: Reinforcement Learning for Constructive Safety Alignment in Large Language Models","display_name":"Oyster-II: Reinforcement Learning for Constructive Safety Alignment in Large Language Models","publication_year":2026,"publication_date":"2026-07-03","ids":{"openalex":"https://openalex.org/W7167594303","doi":"https://doi.org/10.48550/arxiv.2607.02914"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2607.02914","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2607.02914","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2607.02914","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5020880820","display_name":"Jiyang Guan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Guan, Jiyang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5140222737","display_name":"Yong Xie","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xie, Yong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5140172574","display_name":"Jun Chen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Jun","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5140159188","display_name":"Jiexi Liu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Jiexi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5021782599","display_name":"Z Zhenyu Ye","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ye, Zipeng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5140213271","display_name":"Defeng Li","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Defeng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5140154304","display_name":"Jiayu Shen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shen, Jiayu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5140210056","display_name":"Jialing Tao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tao, Jialing","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5140219645","display_name":"Hui Xue","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xue, Hui","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.295199990272522,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.295199990272522,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.13600000739097595,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.11550000309944153,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/constructive","display_name":"Constructive","score":0.6715999841690063},{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.6682000160217285},{"id":"https://openalex.org/keywords/helpfulness","display_name":"Helpfulness","score":0.5843999981880188},{"id":"https://openalex.org/keywords/generalization","display_name":"Generalization","score":0.5371999740600586},{"id":"https://openalex.org/keywords/term","display_name":"Term (time)","score":0.4122999906539917},{"id":"https://openalex.org/keywords/trustworthiness","display_name":"Trustworthiness","score":0.3221000134944916}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7203999757766724},{"id":"https://openalex.org/C2778701210","wikidata":"https://www.wikidata.org/wiki/Q28130034","display_name":"Constructive","level":3,"score":0.6715999841690063},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.6682000160217285},{"id":"https://openalex.org/C2781265381","wikidata":"https://www.wikidata.org/wiki/Q5710255","display_name":"Helpfulness","level":2,"score":0.5843999981880188},{"id":"https://openalex.org/C177148314","wikidata":"https://www.wikidata.org/wiki/Q170084","display_name":"Generalization","level":2,"score":0.5371999740600586},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4442000091075897},{"id":"https://openalex.org/C61797465","wikidata":"https://www.wikidata.org/wiki/Q1188986","display_name":"Term (time)","level":2,"score":0.4122999906539917},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.3626999855041504},{"id":"https://openalex.org/C112930515","wikidata":"https://www.wikidata.org/wiki/Q4389547","display_name":"Risk analysis (engineering)","level":1,"score":0.33480000495910645},{"id":"https://openalex.org/C153701036","wikidata":"https://www.wikidata.org/wiki/Q659974","display_name":"Trustworthiness","level":2,"score":0.3221000134944916},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.29840001463890076},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.2615000009536743},{"id":"https://openalex.org/C136197465","wikidata":"https://www.wikidata.org/wiki/Q1729295","display_name":"Variety (cybernetics)","level":2,"score":0.25760000944137573},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.251800000667572},{"id":"https://openalex.org/C67203356","wikidata":"https://www.wikidata.org/wiki/Q1321905","display_name":"Reinforcement","level":2,"score":0.2513999938964844}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2607.02914","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2607.02914","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2607.02914","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2607.02914","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"score":0.715216338634491,"display_name":"Peace, Justice and strong institutions","id":"https://metadata.un.org/sdg/16"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Large":[0],"language":[1],"models":[2],"(LLMs)":[3],"have":[4],"demonstrated":[5],"remarkable":[6],"capabilities":[7],"across":[8,141],"diverse":[9],"applications,":[10],"yet":[11],"ensuring":[12],"their":[13],"simultaneous":[14],"safety,":[15],"helpfulness,":[16],"and":[17,45,90,111,149,162],"trustworthiness":[18],"remains":[19],"a":[20,91,121,131,136],"persistent":[21],"challenge.":[22],"Conventional":[23],"refusal-oriented":[24],"alignment":[25,127],"strategies":[26],"mitigate":[27],"harmful":[28],"content":[29],"generation":[30],"but":[31],"systematically":[32],"fail":[33],"to":[34,87,106,160],"serve":[35],"legitimate":[36],"user":[37,112],"needs,":[38],"often":[39],"withholding":[40],"information":[41],"that":[42,129],"could":[43],"safely":[44],"constructively":[46],"address":[47,115],"the":[48,56],"underlying":[49],"intent":[50],"of":[51,78],"sensitive":[52],"queries.":[53],"Building":[54],"upon":[55],"constructive":[57,125],"safety":[58,71,85,95,126,154],"paradigm":[59,133],"pioneered":[60],"by":[61],"Oyster-I,":[62],"which":[63],"moves":[64],"beyond":[65],"blanket":[66],"refusal":[67],"toward":[68],"thoughtful,":[69],"response-oriented":[70],"alignment,":[72],"we":[73,93,118],"identify":[74],"two":[75],"critical":[76],"limitations":[77],"its":[79,150],"Supervised":[80],"Fine-Tuning":[81],"(SFT)-based":[82],"scheme:":[83],"insufficient":[84],"generalization":[86],"out-of-distribution":[88],"scenarios":[89],"phenomenon":[92],"term":[94],"chain-of-thought":[96],"(CoT)":[97],"over-generalization,":[98],"wherein":[99],"safety-oriented":[100],"reasoning":[101],"patterns":[102],"are":[103],"excessively":[104],"applied":[105],"benign":[107],"queries,":[108],"degrading":[109],"helpfulness":[110],"experience.":[113],"To":[114],"these":[116],"limitations,":[117],"propose":[119],"Oyster-II,":[120],"reinforcement":[122,138],"learning":[123,139],"(RL)-based":[124],"framework":[128],"adopts":[130],"Zero-RL":[132],"combined":[134],"with":[135],"multi-stage":[137],"strategy.Evaluated":[140],"extensive":[142],"benchmarks,":[143],"Oyster-II":[144],"comprehensively":[145],"surpasses":[146],"both":[147],"Qwen3-14B":[148],"predecessor":[151],"Oyster-I":[152],"on":[153],"dimensions,":[155],"achieving":[156],"cross-scale":[157],"performance":[158],"comparable":[159],"Qwen3-Max":[161],"Qwen3.5-397B.":[163]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-07-08T00:00:00"}