{"id":"https://openalex.org/W4415961936","doi":"https://doi.org/10.48550/arxiv.2510.17139","title":"Rethinking On-policy Optimization for Query Augmentation","display_name":"Rethinking On-policy Optimization for Query Augmentation","publication_year":2025,"publication_date":"2025-10-20","ids":{"openalex":"https://openalex.org/W4415961936","doi":"https://doi.org/10.48550/arxiv.2510.17139"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2510.17139","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2510.17139","pdf_url":"https://arxiv.org/pdf/2510.17139","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2510.17139","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5036683049","display_name":"Zhichao Xu","orcid":"https://orcid.org/0000-0002-2370-4487","observed_orcids":["https://orcid.org/0000-0002-2370-4487","https://orcid.org/0009-0002-2475-2031"]},"institutions":[{"id":"https://openalex.org/I223532165","display_name":"University of Utah","ror":"https://ror.org/03r0ha626","country_code":"US","type":"education","lineage":["https://openalex.org/I223532165"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Xu, Zhichao","raw_affiliation_strings":["University of Utah ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Utah ,","institution_ids":["https://openalex.org/I223532165"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5012958162","display_name":"Shengyao Zhuang","orcid":"https://orcid.org/0000-0002-6711-0955","observed_orcids":["https://orcid.org/0000-0002-6711-0955"]},"institutions":[{"id":"https://openalex.org/I165143802","display_name":"The University of Queensland","ror":"https://ror.org/00rqy9422","country_code":"AU","type":"education","lineage":["https://openalex.org/I165143802"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Zhuang, Shengyao","raw_affiliation_strings":["University of Queensland ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Queensland ,","institution_ids":["https://openalex.org/I165143802"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5021623012","display_name":"Xueguang Ma","orcid":"https://orcid.org/0000-0003-3430-4910","observed_orcids":["https://orcid.org/0000-0003-3430-4910"]},"institutions":[{"id":"https://openalex.org/I151746483","display_name":"University of Waterloo","ror":"https://ror.org/01aff2v68","country_code":"CA","type":"education","lineage":["https://openalex.org/I151746483"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Ma, Xueguang","raw_affiliation_strings":["University of Waterloo ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Waterloo ,","institution_ids":["https://openalex.org/I151746483"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Chen, Bingsen","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I57206974","display_name":"New York University","ror":"https://ror.org/0190ak572","country_code":"US","type":"education","lineage":["https://openalex.org/I57206974"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Chen, Bingsen","raw_affiliation_strings":["New York University ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"New York University ,","institution_ids":["https://openalex.org/I57206974"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5057838053","display_name":"Yijun Tian","orcid":"https://orcid.org/0000-0003-2795-6080","observed_orcids":["https://orcid.org/0000-0003-2795-6080"]},"institutions":[{"id":"https://openalex.org/I107639228","display_name":"University of Notre Dame","ror":"https://ror.org/00mkhxb43","country_code":"US","type":"education","lineage":["https://openalex.org/I107639228"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Tian, Yijun","raw_affiliation_strings":["University of Notre Dame ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Notre Dame ,","institution_ids":["https://openalex.org/I107639228"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5048033573","display_name":"Fengran Mo","orcid":"https://orcid.org/0000-0002-0838-6994","observed_orcids":["https://orcid.org/0000-0002-0838-6994"]},"institutions":[{"id":"https://openalex.org/I70931966","display_name":"Universit\u00e9 de Montr\u00e9al","ror":"https://ror.org/0161xgx34","country_code":"CA","type":"education","lineage":["https://openalex.org/I70931966"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Mo, Fengran","raw_affiliation_strings":["Universit\u00e9 de Montr\u00e9al ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Universit\u00e9 de Montr\u00e9al ,","institution_ids":["https://openalex.org/I70931966"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Li, Tao","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Tao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101713293","display_name":"Jie Cao","orcid":"https://orcid.org/0000-0002-5255-5927","observed_orcids":["https://orcid.org/0000-0002-5255-5927"]},"institutions":[{"id":"https://openalex.org/I8692664","display_name":"University of Oklahoma","ror":"https://ror.org/02aqsxs83","country_code":"US","type":"education","lineage":["https://openalex.org/I8692664"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Cao, Jie","raw_affiliation_strings":["University of Oklahoma"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Oklahoma","institution_ids":["https://openalex.org/I8692664"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5013135203","display_name":"Vivek Srikumar","orcid":"https://orcid.org/0000-0003-0419-6568","observed_orcids":["https://orcid.org/0000-0003-0419-6568"]},"institutions":[{"id":"https://openalex.org/I223532165","display_name":"University of Utah","ror":"https://ror.org/03r0ha626","country_code":"US","type":"education","lineage":["https://openalex.org/I223532165"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Srikumar, Vivek","raw_affiliation_strings":["University of Utah ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Utah ,","institution_ids":["https://openalex.org/I223532165"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":7,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":3,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.000699999975040555,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10203","display_name":"Recommender Systems and Techniques","score":9.999999747378752e-05,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/query-expansion","display_name":"query expansion","score":0.996999979019165},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.7559999823570251},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"information retrieval","score":0.8989999890327454},{"id":"https://openalex.org/keywords/tool-retrieval","display_name":"tool retrieval","score":0.7020000219345093},{"id":"https://openalex.org/keywords/ad-hoc-retrieval","display_name":"ad hoc retrieval","score":0.9229999780654907}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7953000068664551},{"id":"https://openalex.org/C99016210","wikidata":"https://www.wikidata.org/wiki/Q5488129","display_name":"Query expansion","level":2,"score":0.6985999941825867},{"id":"https://openalex.org/C157692150","wikidata":"https://www.wikidata.org/wiki/Q2919848","display_name":"Query optimization","level":2,"score":0.5922999978065491},{"id":"https://openalex.org/C2780598303","wikidata":"https://www.wikidata.org/wiki/Q65921492","display_name":"Flexibility (engineering)","level":2,"score":0.5530999898910522},{"id":"https://openalex.org/C192028432","wikidata":"https://www.wikidata.org/wiki/Q845739","display_name":"Query language","level":2,"score":0.5457000136375427},{"id":"https://openalex.org/C154690210","wikidata":"https://www.wikidata.org/wiki/Q1668499","display_name":"Rewriting","level":2,"score":0.5325000286102295},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.5220999717712402},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.43959999084472656},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.4162999987602234},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.41029998660087585},{"id":"https://openalex.org/C39890363","wikidata":"https://www.wikidata.org/wiki/Q36108","display_name":"Generative grammar","level":2,"score":0.4059999883174896},{"id":"https://openalex.org/C96956885","wikidata":"https://www.wikidata.org/wiki/Q6138701","display_name":"RDF query language","level":5,"score":0.4018000066280365},{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.39469999074935913},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3472999930381775},{"id":"https://openalex.org/C118689300","wikidata":"https://www.wikidata.org/wiki/Q7978614","display_name":"Web query classification","level":4,"score":0.3319000005722046},{"id":"https://openalex.org/C164120249","wikidata":"https://www.wikidata.org/wiki/Q995982","display_name":"Web search query","level":3,"score":0.31049999594688416},{"id":"https://openalex.org/C117251300","wikidata":"https://www.wikidata.org/wiki/Q1849855","display_name":"Parametric statistics","level":2,"score":0.2976999878883362},{"id":"https://openalex.org/C192939062","wikidata":"https://www.wikidata.org/wiki/Q104840822","display_name":"Sargable","level":4,"score":0.29510000348091125},{"id":"https://openalex.org/C61797465","wikidata":"https://www.wikidata.org/wiki/Q1188986","display_name":"Term (time)","level":2,"score":0.29319998621940613},{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.289000004529953},{"id":"https://openalex.org/C24028149","wikidata":"https://www.wikidata.org/wiki/Q7094056","display_name":"Online aggregation","level":5,"score":0.25380000472068787}],"mesh":[],"locations_count":3,"locations":[{"id":"pmh:oai:arXiv.org:2510.17139","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2510.17139","pdf_url":"https://arxiv.org/pdf/2510.17139","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:shareok.org:11244/342956","doi":null,"is_oa":true,"landing_page_url":"https://shareok.org/handle/11244/342956","pdf_url":"https://shareok.org/bitstreams/f1e07cc2-45ad-4416-8543-ff4d3a94865e/download","source":{"id":"https://openalex.org/S4306402274","display_name":"SHAREOK (University of Oklahoma; Oklahoma State University; Central Oklahoma University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"},{"id":"doi:10.48550/arxiv.2510.17139","doi":"https://doi.org/10.48550/arxiv.2510.17139","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2510.17139","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2510.17139","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2510.17139","pdf_url":"https://arxiv.org/pdf/2510.17139","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4415961936.pdf","grobid_xml":"https://content.openalex.org/works/W4415961936.grobid-xml"},"referenced_works_count":52,"referenced_works":["https://openalex.org/W2612431505","https://openalex.org/W2736601468","https://openalex.org/W2789566302","https://openalex.org/W2889787757","https://openalex.org/W2912924812","https://openalex.org/W2951534261","https://openalex.org/W2963748441","https://openalex.org/W3011794880","https://openalex.org/W3023035014","https://openalex.org/W3035169992","https://openalex.org/W3043590089","https://openalex.org/W3133024802","https://openalex.org/W3153794000","https://openalex.org/W4310923309","https://openalex.org/W4385565351","https://openalex.org/W4385569686","https://openalex.org/W4385573600","https://openalex.org/W4389518671","https://openalex.org/W4389518951","https://openalex.org/W4389520758","https://openalex.org/W4391631327","https://openalex.org/W4393300120","https://openalex.org/W4401660192","https://openalex.org/W4402670301","https://openalex.org/W4404780820","https://openalex.org/W4404987838","https://openalex.org/W4405767463","https://openalex.org/W4407820180","https://openalex.org/W4411119048","https://openalex.org/W4411638747","https://openalex.org/W4412673538","https://openalex.org/W4412886806","https://openalex.org/W4414281281","https://openalex.org/W4414939355","https://openalex.org/W4416033948","https://openalex.org/W4416035553","https://openalex.org/W4416068048","https://openalex.org/W4416176141","https://openalex.org/W4416256501","https://openalex.org/W4417065331","https://openalex.org/W4417252869","https://openalex.org/W4417422257","https://openalex.org/W6910528035","https://openalex.org/W6929725739","https://openalex.org/W6966728296","https://openalex.org/W7075994024","https://openalex.org/W7077885508","https://openalex.org/W7090728590","https://openalex.org/W7106767948","https://openalex.org/W7117162136","https://openalex.org/W7167943452","https://openalex.org/W7168424163"],"related_works":["https://openalex.org/W7163871828","https://openalex.org/W7133331411","https://openalex.org/W4416036130","https://openalex.org/W7171684364","https://openalex.org/W4307782988","https://openalex.org/W7214992185","https://openalex.org/W4413145989","https://openalex.org/W4410812983","https://openalex.org/W4402684013","https://openalex.org/W4404782160"],"abstract_inverted_index":{"Recent":[0],"advances":[1],"in":[2,14,154],"large":[3],"language":[4],"models":[5],"(LLMs)":[6],"have":[7,24,75],"led":[8],"to":[9,30,56,160,212],"a":[10,114,145,162,200],"surge":[11],"of":[12,92,180,186],"interest":[13],"query":[15,60,96,120],"augmentation":[16,97,121],"for":[17,59],"information":[18],"retrieval":[19,64,166],"(IR).":[20],"Two":[21],"main":[22],"approaches":[23,74],"emerged.":[25],"The":[26,50],"first":[27,89],"prompts":[28],"LLMs":[29,58],"generate":[31,161],"answers":[32],"or":[33,47,127],"pseudo-documents":[34],"that":[35,112,164,199],"serve":[36],"as":[37],"new":[38],"queries,":[39],"relying":[40],"purely":[41],"on":[42,124],"the":[43,72,88,156,171,175,183,204],"model's":[44],"parametric":[45],"knowledge":[46],"contextual":[48],"information.":[49],"second":[51],"applies":[52],"reinforcement":[53],"learning":[54],"(RL)":[55],"fine-tune":[57],"rewriting,":[61,197],"directly":[62],"optimizing":[63],"metrics.":[65],"While":[66],"having":[67],"respective":[68],"advantages":[69],"and":[70,94,105,177,195],"limitations,":[71],"two":[73],"not":[76],"been":[77],"compared":[78],"under":[79,113],"consistent":[80],"experimental":[81],"conditions.":[82],"In":[83],"this":[84,141],"work,":[85],"we":[86,143],"present":[87],"systematic":[90],"comparison":[91,116],"prompting-based":[93],"RL-based":[95,132,196],"across":[98],"diverse":[99],"benchmarks,":[100],"including":[101],"evidence-seeking,":[102],"ad":[103],"hoc,":[104],"tool":[106],"retrieval.":[107],"Our":[108],"key":[109],"finding":[110],"is":[111],"compute-aware":[115],"setting,":[117],"simple,":[118],"training-free":[119],"often":[122],"performs":[123],"par":[125],"with,":[126],"even":[128],"surpasses,":[129],"more":[130],"expensive":[131],"counterparts,":[133],"especially":[134],"when":[135],"using":[136],"powerful":[137],"LLMs.":[138],"Motivated":[139],"by":[140],"discovery,":[142],"introduce":[144],"novel":[146],"hybrid":[147],"method,":[148],"On-policy":[149],"Pseudo-document":[150],"Query":[151],"Expansion":[152],"(OPQE),":[153],"which":[155],"LLM":[157],"policy":[158],"learns":[159],"pseudo-document":[163],"maximizes":[165],"performance,":[167],"rather":[168],"than":[169],"rewriting":[170],"query,":[172],"thus":[173],"merging":[174],"flexibility":[176],"generative":[178],"structure":[179],"prompting":[181,194],"with":[182],"targeted":[184],"optimization":[185],"RL.":[187],"We":[188,207],"show":[189],"OPQE":[190],"outperforms":[191],"both":[192],"standalone":[193],"demonstrating":[198],"synergistic":[201],"approach":[202],"yields":[203],"best":[205],"results.":[206],"open":[208],"source":[209],"our":[210],"implementation":[211],"facilitate":[213],"reproducibility.":[214]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":1}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2025-10-22T00:00:00"}
