{"id":"https://openalex.org/W7164031432","doi":"https://doi.org/10.48550/arxiv.2606.09635","title":"Gradient-Guided Reward Optimization for Inference-time Alignment","display_name":"Gradient-Guided Reward Optimization for Inference-time Alignment","publication_year":2026,"publication_date":"2026-06-08","ids":{"openalex":"https://openalex.org/W7164031432","doi":"https://doi.org/10.48550/arxiv.2606.09635"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.09635","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.09635","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.09635","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5036129594","display_name":"Hankun Lin","orcid":"https://orcid.org/0000-0002-5441-5178"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lin, Hankun","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5138257315","display_name":"Ruqi Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Ruqi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.20829999446868896,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.20829999446868896,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12761","display_name":"Data Stream Mining Techniques","score":0.16200000047683716,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.15189999341964722,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/robustness","display_name":"Robustness (evolution)","score":0.6902999877929688},{"id":"https://openalex.org/keywords/entropy","display_name":"Entropy (arrow of time)","score":0.5117999911308289},{"id":"https://openalex.org/keywords/importance-sampling","display_name":"Importance sampling","score":0.46140000224113464},{"id":"https://openalex.org/keywords/imperfect","display_name":"Imperfect","score":0.4397999942302704},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.4099999964237213},{"id":"https://openalex.org/keywords/bounded-function","display_name":"Bounded function","score":0.4043999910354614},{"id":"https://openalex.org/keywords/cross-entropy","display_name":"Cross entropy","score":0.4041000008583069},{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.4020000100135803}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6924999952316284},{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.6902999877929688},{"id":"https://openalex.org/C106301342","wikidata":"https://www.wikidata.org/wiki/Q4117933","display_name":"Entropy (arrow of time)","level":2,"score":0.5117999911308289},{"id":"https://openalex.org/C52740198","wikidata":"https://www.wikidata.org/wiki/Q1539564","display_name":"Importance sampling","level":3,"score":0.46140000224113464},{"id":"https://openalex.org/C2780310539","wikidata":"https://www.wikidata.org/wiki/Q12547192","display_name":"Imperfect","level":2,"score":0.4397999942302704},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.4099999964237213},{"id":"https://openalex.org/C34388435","wikidata":"https://www.wikidata.org/wiki/Q2267362","display_name":"Bounded function","level":2,"score":0.4043999910354614},{"id":"https://openalex.org/C167981619","wikidata":"https://www.wikidata.org/wiki/Q1685498","display_name":"Cross entropy","level":3,"score":0.4041000008583069},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.4020000100135803},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3643999993801117},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.3587999939918518},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.34860000014305115},{"id":"https://openalex.org/C57273362","wikidata":"https://www.wikidata.org/wiki/Q576722","display_name":"Decoding methods","level":2,"score":0.3481000065803528},{"id":"https://openalex.org/C126042441","wikidata":"https://www.wikidata.org/wiki/Q1324888","display_name":"Frame (networking)","level":2,"score":0.33719998598098755},{"id":"https://openalex.org/C140779682","wikidata":"https://www.wikidata.org/wiki/Q210868","display_name":"Sampling (signal processing)","level":3,"score":0.3257000148296356},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.32519999146461487},{"id":"https://openalex.org/C13662910","wikidata":"https://www.wikidata.org/wiki/Q193139","display_name":"Trajectory","level":2,"score":0.2994000017642975},{"id":"https://openalex.org/C149441793","wikidata":"https://www.wikidata.org/wiki/Q200726","display_name":"Probability distribution","level":2,"score":0.29089999198913574},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.28380000591278076},{"id":"https://openalex.org/C137836250","wikidata":"https://www.wikidata.org/wiki/Q984063","display_name":"Optimization problem","level":2,"score":0.2825999855995178},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.27570000290870667},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.27059999108314514},{"id":"https://openalex.org/C127705205","wikidata":"https://www.wikidata.org/wiki/Q5748245","display_name":"Heuristics","level":2,"score":0.25609999895095825}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.09635","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.09635","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.09635","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.09635","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/16","display_name":"Peace, Justice and strong institutions","score":0.439891517162323}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Ensuring":[0],"the":[1,29,46,121],"reliability":[2],"of":[3,98,147],"Large":[4],"Language":[5],"Models":[6],"(LLMs)":[7],"under":[8],"distribution":[9],"drift":[10,99],"requires":[11],"inference-time":[12,15,76,135],"adaptation.":[13],"While":[14],"alignment":[16,136],"methods":[17],"such":[18],"as":[19,31],"Best-of-$N$":[20],"and":[21,51,140,150],"rejection":[22],"sampling":[23],"are":[24],"widely":[25],"used,":[26],"they":[27],"frame":[28],"task":[30],"a":[32,74],"sampling-intensive,":[33],"reward-guided":[34],"search,":[35],"leading":[36],"to":[37,61,93,119,152],"two":[38],"key":[39],"limitations:":[40],"their":[41,52],"performance":[42],"is":[43,160],"bounded":[44],"by":[45,106],"base":[47],"model's":[48],"generation":[49,122],"quality,":[50],"reliance":[53],"on":[54],"imperfect":[55],"reward":[56,62,117,153],"models":[57],"makes":[58],"them":[59],"vulnerable":[60],"hacking.":[63],"To":[64],"address":[65],"these":[66],"challenges,":[67],"we":[68],"introduce":[69],"Gradient-Guided":[70],"Reward":[71],"Optimization":[72],"(GGRO),":[73],"lightweight":[75],"method":[77],"that":[78,131],"performs":[79],"targeted,":[80],"minimal":[81,156],"intervention":[82],"during":[83],"decoding":[84],"via":[85],"gradient":[86,112],"guidance.":[87],"Specifically,":[88],"GGRO":[89,132],"monitors":[90],"token-level":[91],"entropy":[92],"identify":[94],"high-uncertainty":[95],"regions":[96],"indicative":[97],"or":[100],"misalignment.":[101],"Upon":[102],"detection,":[103],"it":[104],"responds":[105],"injecting":[107],"nudging":[108],"tokens,":[109],"generated":[110],"using":[111],"signals":[113],"from":[114],"an":[115],"off-the-shelf":[116],"model,":[118],"steer":[120],"trajectory":[123],"rather":[124],"than":[125],"merely":[126],"re-ranking":[127],"samples.":[128],"Experiments":[129],"show":[130],"consistently":[133],"improves":[134],"across":[137],"safety,":[138],"helpfulness,":[139],"reasoning":[141],"benchmarks.":[142],"It":[143],"also":[144],"increases":[145],"coverage":[146],"high-quality":[148],"responses":[149],"robustness":[151],"hacking,":[154],"with":[155],"computational":[157],"overhead.":[158],"Code":[159],"available":[161],"at":[162],"https://github.com/lhk2004/GGRO.":[163]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-06-10T00:00:00"}