{"id":"https://openalex.org/W7162092528","doi":"https://doi.org/10.1145/3786335.3813164","title":"Composing Policy Gradients and Prompt Optimization for Language Model Programs","display_name":"Composing Policy Gradients and Prompt Optimization for Language Model Programs","publication_year":2026,"publication_date":"2026-05-22","ids":{"openalex":"https://openalex.org/W7162092528","doi":"https://doi.org/10.1145/3786335.3813164"},"language":null,"primary_location":{"id":"doi:10.1145/3786335.3813164","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3786335.3813164","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM Conference on AI and Agentic Systems","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3786335.3813164","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5032058128","display_name":"Noah Ziems","orcid":null},"institutions":[{"id":"https://openalex.org/I107639228","display_name":"University of Notre Dame","ror":"https://ror.org/00mkhxb43","country_code":"US","type":"education","lineage":["https://openalex.org/I107639228"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Noah Ziems","raw_affiliation_strings":["MIT, Cambridge, USA and University of Notre Dame, South Bend, USA"],"raw_orcid":"https://orcid.org/0009-0002-1002-1959","affiliations":[{"raw_affiliation_string":"MIT, Cambridge, USA and University of Notre Dame, South Bend, USA","institution_ids":["https://openalex.org/I107639228"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5088639037","display_name":"Dilara Soylu","orcid":null},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Dilara Soylu","raw_affiliation_strings":["Stanford University, Stanford, USA"],"raw_orcid":"https://orcid.org/0009-0003-6265-2055","affiliations":[{"raw_affiliation_string":"Stanford University, Stanford, USA","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136777292","display_name":"Lakshya A Agrawal","orcid":"https://orcid.org/0000-0003-0409-8212"},"institutions":[{"id":"https://openalex.org/I134446601","display_name":"Berkeley College","ror":"https://ror.org/02xewxa75","country_code":"US","type":"education","lineage":["https://openalex.org/I134446601"]},{"id":"https://openalex.org/I95457486","display_name":"University of California, Berkeley","ror":"https://ror.org/01an7q238","country_code":"US","type":"education","lineage":["https://openalex.org/I95457486"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Lakshya A Agrawal","raw_affiliation_strings":["UC Berkeley, Berkeley, USA"],"raw_orcid":"https://orcid.org/0000-0003-0409-8212","affiliations":[{"raw_affiliation_string":"UC Berkeley, Berkeley, USA","institution_ids":["https://openalex.org/I134446601","https://openalex.org/I95457486"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136730685","display_name":"Isaac Miller","orcid":"https://orcid.org/0009-0000-5978-4901"},"institutions":[{"id":"https://openalex.org/I21160419","display_name":"Ansys (United States)","ror":"https://ror.org/05cf5b117","country_code":"US","type":"company","lineage":["https://openalex.org/I21160419"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Isaac Miller","raw_affiliation_strings":["Anyscale, New York, USA"],"raw_orcid":"https://orcid.org/0009-0000-5978-4901","affiliations":[{"raw_affiliation_string":"Anyscale, New York, USA","institution_ids":["https://openalex.org/I21160419"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5013691122","display_name":"L. Lai","orcid":null},"institutions":[{"id":"https://openalex.org/I134446601","display_name":"Berkeley College","ror":"https://ror.org/02xewxa75","country_code":"US","type":"education","lineage":["https://openalex.org/I134446601"]},{"id":"https://openalex.org/I95457486","display_name":"University of California, Berkeley","ror":"https://ror.org/01an7q238","country_code":"US","type":"education","lineage":["https://openalex.org/I95457486"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Liheng Lai","raw_affiliation_strings":["UC Berkeley, Berkeley, USA"],"raw_orcid":"https://orcid.org/0009-0004-7429-8062","affiliations":[{"raw_affiliation_string":"UC Berkeley, Berkeley, USA","institution_ids":["https://openalex.org/I134446601","https://openalex.org/I95457486"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136790118","display_name":"Chen Qian","orcid":"https://orcid.org/0009-0006-1481-5069"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen Qian","raw_affiliation_strings":["CMU, Pittsburgh, USA"],"raw_orcid":"https://orcid.org/0009-0006-1481-5069","affiliations":[{"raw_affiliation_string":"CMU, Pittsburgh, USA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136800714","display_name":"Kaiqiang Song","orcid":"https://orcid.org/0009-0007-2007-2596"},"institutions":[{"id":"https://openalex.org/I174135032","display_name":"Bellevue College","ror":"https://ror.org/05gr4yv49","country_code":"US","type":"education","lineage":["https://openalex.org/I174135032"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kaiqiang Song","raw_affiliation_strings":["Zoom, Inc., Bellevue, USA"],"raw_orcid":"https://orcid.org/0009-0007-2007-2596","affiliations":[{"raw_affiliation_string":"Zoom, Inc., Bellevue, USA","institution_ids":["https://openalex.org/I174135032"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136728061","display_name":"Meng Jiang","orcid":"https://orcid.org/0000-0002-3009-519X"},"institutions":[{"id":"https://openalex.org/I107639228","display_name":"University of Notre Dame","ror":"https://ror.org/00mkhxb43","country_code":"US","type":"education","lineage":["https://openalex.org/I107639228"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Meng Jiang","raw_affiliation_strings":["University of Notre Dame, South Bend, USA"],"raw_orcid":"https://orcid.org/0000-0002-3009-519X","affiliations":[{"raw_affiliation_string":"University of Notre Dame, South Bend, USA","institution_ids":["https://openalex.org/I107639228"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5004921249","display_name":"Dan Klein","orcid":"https://orcid.org/0000-0002-8881-1902"},"institutions":[{"id":"https://openalex.org/I134446601","display_name":"Berkeley College","ror":"https://ror.org/02xewxa75","country_code":"US","type":"education","lineage":["https://openalex.org/I134446601"]},{"id":"https://openalex.org/I95457486","display_name":"University of California, Berkeley","ror":"https://ror.org/01an7q238","country_code":"US","type":"education","lineage":["https://openalex.org/I95457486"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Dan Klein","raw_affiliation_strings":["UC Berkeley, Berkeley, USA"],"raw_orcid":"https://orcid.org/0009-0002-5983-2904","affiliations":[{"raw_affiliation_string":"UC Berkeley, Berkeley, USA","institution_ids":["https://openalex.org/I134446601","https://openalex.org/I95457486"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136779894","display_name":"Matei Zaharia","orcid":"https://orcid.org/0000-0002-7547-7204"},"institutions":[{"id":"https://openalex.org/I134446601","display_name":"Berkeley College","ror":"https://ror.org/02xewxa75","country_code":"US","type":"education","lineage":["https://openalex.org/I134446601"]},{"id":"https://openalex.org/I95457486","display_name":"University of California, Berkeley","ror":"https://ror.org/01an7q238","country_code":"US","type":"education","lineage":["https://openalex.org/I95457486"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Matei Zaharia","raw_affiliation_strings":["UC Berkeley, Berkeley, USA"],"raw_orcid":"https://orcid.org/0000-0002-7547-7204","affiliations":[{"raw_affiliation_string":"UC Berkeley, Berkeley, USA","institution_ids":["https://openalex.org/I134446601","https://openalex.org/I95457486"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5031474902","display_name":"Karel D\u2019Oosterlinck","orcid":null},"institutions":[{"id":"https://openalex.org/I4210128771","display_name":"Contextual Change (United States)","ror":"https://ror.org/03bskcm82","country_code":"US","type":"company","lineage":["https://openalex.org/I4210128771"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Karel D\u2019Oosterlinck","raw_affiliation_strings":["Contextual AI, Mountain View, USA"],"raw_orcid":"https://orcid.org/0000-0003-1695-1014","affiliations":[{"raw_affiliation_string":"Contextual AI, Mountain View, USA","institution_ids":["https://openalex.org/I4210128771"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133826478","display_name":"Christopher Potts","orcid":null},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Christopher Potts","raw_affiliation_strings":["Stanford University, Stanford, USA"],"raw_orcid":"https://orcid.org/0000-0002-7978-6055","affiliations":[{"raw_affiliation_string":"Stanford University, Stanford, USA","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5130338108","display_name":"Omar Khattab","orcid":null},"institutions":[{"id":"https://openalex.org/I4210110987","display_name":"IIT@MIT","ror":"https://ror.org/01wp8zh54","country_code":"US","type":"facility","lineage":["https://openalex.org/I30771326","https://openalex.org/I4210110987"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Omar Khattab","raw_affiliation_strings":["MIT, Cambridge, USA"],"raw_orcid":"https://orcid.org/0000-0002-9836-4413","affiliations":[{"raw_affiliation_string":"MIT, Cambridge, USA","institution_ids":["https://openalex.org/I4210110987"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":8,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"197","last_page":"213"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.27219998836517334,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.27219998836517334,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.11670000106096268,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.10419999808073044,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/leverage","display_name":"Leverage (statistics)","score":0.7267000079154968},{"id":"https://openalex.org/keywords/modular-design","display_name":"Modular design","score":0.5024999976158142},{"id":"https://openalex.org/keywords/delegation","display_name":"Delegation","score":0.4408999979496002},{"id":"https://openalex.org/keywords/natural-language","display_name":"Natural language","score":0.41830000281333923},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.3937000036239624},{"id":"https://openalex.org/keywords/natural-language-generation","display_name":"Natural language generation","score":0.36000001430511475},{"id":"https://openalex.org/keywords/template","display_name":"Template","score":0.326200008392334}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7903000116348267},{"id":"https://openalex.org/C153083717","wikidata":"https://www.wikidata.org/wiki/Q6535263","display_name":"Leverage (statistics)","level":2,"score":0.7267000079154968},{"id":"https://openalex.org/C101468663","wikidata":"https://www.wikidata.org/wiki/Q1620158","display_name":"Modular design","level":2,"score":0.5024999976158142},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.45399999618530273},{"id":"https://openalex.org/C86532276","wikidata":"https://www.wikidata.org/wiki/Q1184065","display_name":"Delegation","level":2,"score":0.4408999979496002},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.41830000281333923},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.40310001373291016},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.3937000036239624},{"id":"https://openalex.org/C2776187449","wikidata":"https://www.wikidata.org/wiki/Q1513879","display_name":"Natural language generation","level":3,"score":0.36000001430511475},{"id":"https://openalex.org/C82714645","wikidata":"https://www.wikidata.org/wiki/Q438331","display_name":"Template","level":2,"score":0.326200008392334},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.31949999928474426},{"id":"https://openalex.org/C179603123","wikidata":"https://www.wikidata.org/wiki/Q1941921","display_name":"Modeling language","level":3,"score":0.3003999888896942},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.29109999537467957},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.2865000069141388},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.27129998803138733},{"id":"https://openalex.org/C137836250","wikidata":"https://www.wikidata.org/wiki/Q984063","display_name":"Optimization problem","level":2,"score":0.27129998803138733},{"id":"https://openalex.org/C2781311116","wikidata":"https://www.wikidata.org/wiki/Q83306","display_name":"Group (periodic table)","level":2,"score":0.2685000002384186},{"id":"https://openalex.org/C165696696","wikidata":"https://www.wikidata.org/wiki/Q11287","display_name":"Exploit","level":2,"score":0.25920000672340393},{"id":"https://openalex.org/C184337299","wikidata":"https://www.wikidata.org/wiki/Q1437428","display_name":"Semantics (computer science)","level":2,"score":0.2508000135421753}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3786335.3813164","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3786335.3813164","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM Conference on AI and Agentic Systems","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3786335.3813164","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3786335.3813164","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM Conference on AI and Agentic Systems","raw_type":"proceedings-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/16","display_name":"Peace, Justice and strong institutions","score":0.4182123839855194}],"awards":[{"id":"https://openalex.org/G3533996891","display_name":null,"funder_award_id":"IIS-2142827","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G4956134732","display_name":null,"funder_award_id":"IIS-2234058","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":14,"referenced_works":["https://openalex.org/W2889787757","https://openalex.org/W3045492832","https://openalex.org/W3098267758","https://openalex.org/W3101682885","https://openalex.org/W3210877910","https://openalex.org/W4385573003","https://openalex.org/W4387321091","https://openalex.org/W4389520756","https://openalex.org/W4404782958","https://openalex.org/W4404792910","https://openalex.org/W7133208539","https://openalex.org/W7133208667","https://openalex.org/W7133211372","https://openalex.org/W7133225282"],"related_works":[],"abstract_inverted_index":{"Group":[0],"Relative":[1],"Policy":[2],"Optimization":[3],"(GRPO)":[4],"has":[5],"proven":[6],"to":[7,54,60,70],"be":[8],"an":[9,85],"effective":[10],"tool":[11],"for":[12,74,88,99,125],"post-training":[13],"language":[14],"models":[15],"(LMs).":[16],"However,":[17],"AI":[18],"systems":[19],"are":[20],"increasingly":[21],"expressed":[22],"as":[23,84,119],"modular":[24],"programs":[25,77,90],"that":[26,129],"mix":[27],"together":[28,143],"multiple":[29],"LM":[30,89],"calls":[31],"with":[32,138],"distinct":[33],"prompt":[34,100,140,166],"templates":[35],"and":[36,39,78,95,113,142,155],"other":[37],"tools,":[38],"it":[40,67,80],"is":[41,68],"not":[42],"clear":[43],"how":[44],"practitioners":[45],"can":[46,81],"best":[47],"leverage":[48],"online":[49],"RL":[50],"algorithms":[51],"like":[52],"GRPO":[53,73,107,130],"improve":[55,145],"these":[56],"systems.":[57],"We":[58,123,171],"begin":[59],"address":[61],"this":[62],"challenge":[63],"by":[64,147],"investigating":[65],"whether":[66,79],"possible":[69],"effectively":[71],"instantiate":[72],"arbitrary":[75],"multi-prompt":[76],"work":[82],"robustly":[83],"off-the-shelf":[86],"optimizer":[87],"using":[91],"the":[92,126,160,175],"same":[93],"abstractions":[94],"constraints":[96],"typically":[97],"involved":[98],"optimization.":[101],"Our":[102],"main":[103],"variant":[104],"of":[105],"multi-module":[106,133],"constructs":[108],"groups":[109],"from":[110],"module-level":[111],"invocations,":[112],"we":[114],"also":[115],"consider":[116],"trajectory-level":[117],"grouping":[118],"another":[120],"natural":[121],"instantiation.":[122],"find":[124],"first":[127],"time":[128],"(and":[131],"its":[132,169],"counterpart)":[134],"empirically":[135],"composes":[136],"well":[137],"automatic":[139],"optimization,":[141],"they":[144],"accuracy":[146],"11%":[148],"on":[149,168],"average":[150],"across":[151],"classification,":[152],"many-hop":[153],"search,":[154],"privacy-preserving":[156],"delegation":[157],"tasks":[158],"against":[159,165],"post-trained":[161],"LM\u2014with":[162],"5%":[163],"gains":[164],"optimization":[167],"own.":[170],"open-source":[172],"mmGRPO":[173],"in":[174],"DSPy":[176],"library":[177],"at":[178],"dspy.ai.":[179]},"counts_by_year":[],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2026-05-23T00:00:00"}