{"id":"https://openalex.org/W7139008821","doi":"https://doi.org/10.48550/arxiv.2603.16152","title":"HIPO: Instruction Hierarchy via Constrained Reinforcement Learning","display_name":"HIPO: Instruction Hierarchy via Constrained Reinforcement Learning","publication_year":2026,"publication_date":"2026-03-17","ids":{"openalex":"https://openalex.org/W7139008821","doi":"https://doi.org/10.48550/arxiv.2603.16152"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2603.16152","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.16152","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2603.16152","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5129777129","display_name":"Keru Chen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Keru","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5129764336","display_name":"Jun Luo","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Luo, Jun","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5006448949","display_name":"Sen Lin","orcid":"https://orcid.org/0000-0002-3797-3215"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lin, Sen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5130054827","display_name":"Yingbin Liang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liang, Yingbin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5129999719","display_name":"Alvaro Velasquez","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Velasquez, Alvaro","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5032194186","display_name":"Nathaniel D. Bastian","orcid":"https://orcid.org/0000-0001-9957-2778"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Bastian, Nathaniel","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5129858942","display_name":"Shaofeng Zou","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zou, Shaofeng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12127","display_name":"Software System Performance and Reliability","score":0.30169999599456787,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12127","display_name":"Software System Performance and Reliability","score":0.30169999599456787,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10703","display_name":"Business Process Modeling and Analysis","score":0.08720000088214874,"subfield":{"id":"https://openalex.org/subfields/1404","display_name":"Management Information Systems"},"field":{"id":"https://openalex.org/fields/14","display_name":"Business, Management and Accounting"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.06350000202655792,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.7379999756813049},{"id":"https://openalex.org/keywords/markov-decision-process","display_name":"Markov decision process","score":0.621399998664856},{"id":"https://openalex.org/keywords/hierarchy","display_name":"Hierarchy","score":0.5544000267982483},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.5076000094413757},{"id":"https://openalex.org/keywords/scheme","display_name":"Scheme (mathematics)","score":0.35920000076293945},{"id":"https://openalex.org/keywords/software-deployment","display_name":"Software deployment","score":0.34040001034736633},{"id":"https://openalex.org/keywords/stability","display_name":"Stability (learning theory)","score":0.337799996137619},{"id":"https://openalex.org/keywords/markov-process","display_name":"Markov process","score":0.320499986410141}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7928000092506409},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.7379999756813049},{"id":"https://openalex.org/C106189395","wikidata":"https://www.wikidata.org/wiki/Q176789","display_name":"Markov decision process","level":3,"score":0.621399998664856},{"id":"https://openalex.org/C31170391","wikidata":"https://www.wikidata.org/wiki/Q188619","display_name":"Hierarchy","level":2,"score":0.5544000267982483},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5281999707221985},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.5076000094413757},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3962000012397766},{"id":"https://openalex.org/C77618280","wikidata":"https://www.wikidata.org/wiki/Q1155772","display_name":"Scheme (mathematics)","level":2,"score":0.35920000076293945},{"id":"https://openalex.org/C105339364","wikidata":"https://www.wikidata.org/wiki/Q2297740","display_name":"Software deployment","level":2,"score":0.34040001034736633},{"id":"https://openalex.org/C112972136","wikidata":"https://www.wikidata.org/wiki/Q7595718","display_name":"Stability (learning theory)","level":2,"score":0.337799996137619},{"id":"https://openalex.org/C159886148","wikidata":"https://www.wikidata.org/wiki/Q176645","display_name":"Markov process","level":2,"score":0.320499986410141},{"id":"https://openalex.org/C165064840","wikidata":"https://www.wikidata.org/wiki/Q1321061","display_name":"Matching (statistics)","level":2,"score":0.32019999623298645},{"id":"https://openalex.org/C49937458","wikidata":"https://www.wikidata.org/wiki/Q2599292","display_name":"Probabilistic logic","level":2,"score":0.3154999911785126},{"id":"https://openalex.org/C98763669","wikidata":"https://www.wikidata.org/wiki/Q176645","display_name":"Markov chain","level":2,"score":0.3041999936103821},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.2930000126361847},{"id":"https://openalex.org/C137836250","wikidata":"https://www.wikidata.org/wiki/Q984063","display_name":"Optimization problem","level":2,"score":0.28220000863075256},{"id":"https://openalex.org/C183322885","wikidata":"https://www.wikidata.org/wiki/Q17007702","display_name":"Context model","level":3,"score":0.26829999685287476},{"id":"https://openalex.org/C55660270","wikidata":"https://www.wikidata.org/wiki/Q5164377","display_name":"Constrained optimization","level":2,"score":0.2597000002861023},{"id":"https://openalex.org/C28901747","wikidata":"https://www.wikidata.org/wiki/Q177571","display_name":"Decision theory","level":2,"score":0.25130000710487366},{"id":"https://openalex.org/C115988155","wikidata":"https://www.wikidata.org/wiki/Q3262192","display_name":"Decision problem","level":2,"score":0.25099998712539673}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2603.16152","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.16152","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2603.16152","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.16152","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"score":0.7095983624458313,"id":"https://metadata.un.org/sdg/16","display_name":"Peace, Justice and strong institutions"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Hierarchical":[0],"Instruction":[1],"Following":[2],"(HIF)":[3],"refers":[4],"to":[5,39,56,92,155],"the":[6,58,62,103,153],"problem":[7,29],"of":[8,17],"prompting":[9],"large":[10],"language":[11],"models":[12],"with":[13],"a":[14,35,71,79,97,164],"priority-ordered":[15],"stack":[16],"instructions.":[18],"Standard":[19],"methods":[20],"like":[21],"RLHF":[22],"and":[23,140],"DPO":[24],"typically":[25],"fail":[26],"in":[27,171],"this":[28,66,119,148],"since":[30],"they":[31],"mainly":[32],"optimize":[33],"for":[34,167],"single":[36],"objective,":[37],"failing":[38],"explicitly":[40],"enforce":[41],"system":[42,86,107,138,161],"prompt":[43,108],"compliance.":[44],"Meanwhile,":[45],"supervised":[46],"fine-tuning":[47],"relies":[48],"on":[49],"mimicking":[50],"filtered,":[51],"compliant":[52],"data,":[53],"which":[54],"fails":[55],"establish":[57],"priority":[59],"asymmetry":[60],"at":[61],"algorithmic":[63,94],"level.":[64],"In":[65],"paper,":[67],"we":[68],"introduce":[69],"\\textsc{HIPO},":[70],"novel":[72],"alignment":[73],"framework":[74],"that":[75,133,147],"formulates":[76],"HIF":[77],"as":[78,110],"Constrained":[80],"Markov":[81],"Decision":[82],"Process.":[83],"\\textsc{HIPO}":[84,134],"elevates":[85],"prompts":[87],"from":[88],"mere":[89],"input":[90],"context":[91],"strict":[93],"boundaries.":[95],"Using":[96],"primal-dual":[98],"safe":[99],"reinforcement":[100],"learning":[101],"approach,":[102],"algorithm":[104],"dynamically":[105],"enforces":[106],"compliance":[109,139],"an":[111],"explicit":[112],"constraint,":[113],"maximizing":[114],"user":[115,141],"utility":[116],"strictly":[117],"within":[118],"feasible":[120],"region.":[121],"Extensive":[122],"evaluations":[123],"across":[124],"diverse":[125],"model":[126,154],"architectures":[127],"(e.g.,":[128],"Qwen,":[129],"Phi,":[130],"Llama)":[131],"demonstrate":[132],"significantly":[135],"improves":[136],"both":[137],"utility.":[142],"Furthermore,":[143],"mechanistic":[144],"analysis":[145],"reveals":[146],"constrained":[149],"optimization":[150],"autonomously":[151],"drives":[152],"shift":[156],"its":[157],"attention":[158],"toward":[159],"long-range":[160],"tokens,":[162],"providing":[163],"principled":[165],"foundation":[166],"reliable":[168],"LLM":[169],"deployment":[170],"complex":[172],"workflows.":[173]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-03-20T00:00:00"}