{"id":"https://openalex.org/W4412889896","doi":"https://doi.org/10.18653/v1/2025.acl-long.1297","title":"Generative Reward Modeling via Synthetic Criteria Preference Learning","display_name":"Generative Reward Modeling via Synthetic Criteria Preference Learning","publication_year":2025,"publication_date":"2025-01-01","ids":{"openalex":"https://openalex.org/W4412889896","doi":"https://doi.org/10.18653/v1/2025.acl-long.1297"},"language":"en","primary_location":{"id":"doi:10.18653/v1/2025.acl-long.1297","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.acl-long.1297","pdf_url":"https://aclanthology.org/2025.acl-long.1297.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 63rd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://aclanthology.org/2025.acl-long.1297.pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5102023923","display_name":"Xiaobo Liang","orcid":"https://orcid.org/0009-0001-1550-2877"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xiaobo Liang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5036979531","display_name":"Haoke Zhang","orcid":"https://orcid.org/0000-0001-7309-2506"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Haoke Zhang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100657514","display_name":"Juntao Li","orcid":"https://orcid.org/0000-0002-6286-7529"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Juntao Li","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5006323375","display_name":"Kehai Chen","orcid":"https://orcid.org/0000-0002-4346-7618"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kehai Chen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102065469","display_name":"Qiaoming Zhu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Qiaoming Zhu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5100671757","display_name":"M. Zhang","orcid":"https://orcid.org/0000-0001-8659-5727"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Min Zhang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"26755","last_page":"26769"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13083","display_name":"Advanced Text Analysis Techniques","score":0.567799985408783,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T13083","display_name":"Advanced Text Analysis Techniques","score":0.567799985408783,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/preference","display_name":"Preference","score":0.6764354109764099},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6751497387886047},{"id":"https://openalex.org/keywords/generative-grammar","display_name":"Generative grammar","score":0.6593511700630188},{"id":"https://openalex.org/keywords/generative-model","display_name":"Generative model","score":0.5796042084693909},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5503520369529724},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.43559694290161133},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.07761558890342712},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.06055575609207153}],"concepts":[{"id":"https://openalex.org/C2781249084","wikidata":"https://www.wikidata.org/wiki/Q908656","display_name":"Preference","level":2,"score":0.6764354109764099},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6751497387886047},{"id":"https://openalex.org/C39890363","wikidata":"https://www.wikidata.org/wiki/Q36108","display_name":"Generative grammar","level":2,"score":0.6593511700630188},{"id":"https://openalex.org/C167966045","wikidata":"https://www.wikidata.org/wiki/Q5532625","display_name":"Generative model","level":3,"score":0.5796042084693909},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5503520369529724},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.43559694290161133},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.07761558890342712},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.06055575609207153}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.18653/v1/2025.acl-long.1297","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.acl-long.1297","pdf_url":"https://aclanthology.org/2025.acl-long.1297.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 63rd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.18653/v1/2025.acl-long.1297","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.acl-long.1297","pdf_url":"https://aclanthology.org/2025.acl-long.1297.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 63rd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G2027526133","display_name":null,"funder_award_id":"BK20220488","funder_id":"https://openalex.org/F4320322769","funder_display_name":"Natural Science Foundation of Jiangsu Province"},{"id":"https://openalex.org/G3114819178","display_name":null,"funder_award_id":"2023QNRC001","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G6441875606","display_name":null,"funder_award_id":"62206194","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320321605","display_name":"Government of Jiangsu Province","ror":"https://ror.org/004svx814"},{"id":"https://openalex.org/F4320322769","display_name":"Natural Science Foundation of Jiangsu Province","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320327518","display_name":"Priority Academic Program Development of Jiangsu Higher Education Institutions","ror":null}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4412889896.pdf","grobid_xml":"https://content.openalex.org/works/W4412889896.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W2961085424","https://openalex.org/W4306674287","https://openalex.org/W4387369504","https://openalex.org/W4365211920","https://openalex.org/W4394896187","https://openalex.org/W3170094116","https://openalex.org/W4386462264","https://openalex.org/W3107602296","https://openalex.org/W3014948380","https://openalex.org/W4364306694"],"abstract_inverted_index":{"Generative":[0],"Reward":[1],"Models":[2],"(GenRMs)":[3],"leverage":[4],"synthesized":[5,42,84,131],"Chains":[6],"of":[7,23,33,54],"Thought":[8],"(CoT)":[9],"to":[10,26,29,58,142],"reduce":[11],"the":[12,27,31,34,76,103,110],"need":[13,111],"for":[14,69,112],"massive":[15],"labeled":[16],"data,":[17],"but":[18],"this":[19,61],"approach":[20],"introduces":[21],"risks":[22],"overoptimization":[24],"due":[25],"inability":[28],"guarantee":[30],"correctness":[32],"CoTs.Identifying":[35],"and":[36,106,152],"optimizing":[37],"unexpected":[38],"behaviors":[39],"within":[40],"these":[41],"CoT":[43,139],"remains":[44],"a":[45,65,79,137],"challenge,":[46],"as":[47],"it":[48],"heavily":[49],"depends":[50],"on":[51,83,123],"precise":[52],"annotations":[53],"intermediate":[55],"behavior,":[56],"similar":[57],"process":[59,97],"supervision.In":[60,114],"work,":[62],"we":[63],"introduce":[64],"criteria-based":[66],"preference":[67,126],"tree":[68,77],"reward":[70,98],"modeling,":[71],"where":[72],"each":[73,86],"path":[74],"in":[75],"represents":[78],"reasoning":[80,87],"trajectory":[81,88],"based":[82],"criteria.Crucially,":[85],"can":[89,133],"be":[90,134],"independently":[91],"optimized":[92],"through":[93],"RL":[94],"algorithm.These":[95],"fine-grained":[96],"signals":[99],"are":[100],"derived":[101],"from":[102],"inferencetime":[104],"computations":[105],"predefined":[107],"rules,":[108],"eliminating":[109],"human":[113,125],"experiments,":[115],"SyncPL":[116],"1":[117],"showed":[118],"significant":[119],"improvements":[120],"over":[121],"baselines":[122],"multiple":[124],"benchmarks.We":[127],"further":[128,146],"demonstrate":[129],"that":[130],"data":[132],"learned":[135],"using":[136],"long":[138],"format,":[140],"analogous":[141],"an":[143],"o1-like":[144],"model,":[145],"enhancing":[147],"performance":[148],"while":[149],"keeping":[150],"stability":[151],"efficiency":[153],"during":[154],"training.":[155]},"counts_by_year":[],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}