{"id":"https://openalex.org/W4403365276","doi":"https://doi.org/10.48550/arxiv.2410.08048","title":"VerifierQ: Enhancing LLM Test Time Compute with Q-Learning-based Verifiers","display_name":"VerifierQ: Enhancing LLM Test Time Compute with Q-Learning-based Verifiers","publication_year":2024,"publication_date":"2024-10-10","ids":{"openalex":"https://openalex.org/W4403365276","doi":"https://doi.org/10.48550/arxiv.2410.08048"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2410.08048","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2410.08048","pdf_url":"https://arxiv.org/pdf/2410.08048","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2410.08048","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5031360369","display_name":"Jianing Qi","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Qi, Jianing","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100703185","display_name":"Hao Tang","orcid":"https://orcid.org/0000-0002-4125-8417"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tang, Hao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5102003535","display_name":"Zhigang Zhu","orcid":"https://orcid.org/0000-0003-4101-4903"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhu, Zhigang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9894999861717224,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9894999861717224,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9639000296592712,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.944599986076355,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/test","display_name":"Test (biology)","score":0.5941654443740845},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.383415162563324},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.34977197647094727},{"id":"https://openalex.org/keywords/ecology","display_name":"Ecology","score":0.07664996385574341},{"id":"https://openalex.org/keywords/biology","display_name":"Biology","score":0.07296812534332275}],"concepts":[{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.5941654443740845},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.383415162563324},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.34977197647094727},{"id":"https://openalex.org/C18903297","wikidata":"https://www.wikidata.org/wiki/Q7150","display_name":"Ecology","level":1,"score":0.07664996385574341},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.07296812534332275}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2410.08048","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2410.08048","pdf_url":"https://arxiv.org/pdf/2410.08048","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2410.08048","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2410.08048","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2410.08048","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2410.08048","pdf_url":"https://arxiv.org/pdf/2410.08048","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G1595963390","display_name":"PFI-RP: Smart and Accessible Transportation Hub for Assistive Navigation and Facility Management","funder_award_id":"1827505","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G8569551686","display_name":"SCC-Planning:  Towards Smart and Accessible Transportation Hub - Research Capacity Building and Community Engagement","funder_award_id":"1737533","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G863362940","display_name":"CISE-MSI: DP: HCC: Training a Virtual Guide Dog for Visually Impaired People to Learn Safe Routes Using Crowdsourcing Multimodal Data","funder_award_id":"2131186","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320332169","display_name":"Directorate for Computer and Information Science and Engineering","ror":"https://ror.org/025kzpk63"},{"id":"https://openalex.org/F4320332467","display_name":"U.S. Air Force","ror":"https://ror.org/006gmme17"},{"id":"https://openalex.org/F4320338279","display_name":"Air Force Office of Scientific Research","ror":"https://ror.org/011e9bt93"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4403365276.pdf","grobid_xml":"https://content.openalex.org/works/W4403365276.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W4396696052","https://openalex.org/W4402327032","https://openalex.org/W2382290278"],"abstract_inverted_index":{"Recent":[0],"advancements":[1,172],"in":[2,32,40,74,158,173,183,203,226],"test":[3],"time":[4],"compute,":[5],"particularly":[6],"through":[7,160],"the":[8,16,29,150,154,210,220],"use":[9],"of":[10,19,164,223],"verifier":[11,38,67,155,168],"models,":[12],"have":[13],"significantly":[14],"enhanced":[15],"reasoning":[17,182,189],"capabilities":[18],"Large":[20],"Language":[21],"Models":[22],"(LLMs).":[23],"This":[24,54,162],"generator-verifier":[25],"approach":[26,60],"closely":[27],"resembles":[28],"actor-critic":[30],"framework":[31],"reinforcement":[33],"learning":[34,50],"(RL).":[35],"However,":[36],"current":[37],"models":[39,169],"LLMs":[41,159],"often":[42],"rely":[43],"on":[44,187],"supervised":[45,198],"fine-tuning":[46,199],"without":[47],"temporal":[48],"difference":[49],"such":[51],"as":[52],"Q-learning.":[53,161],"paper":[55],"introduces":[56,97],"VerifierQ,":[57],"a":[58,98,116],"novel":[59,117],"that":[61],"integrates":[62,115],"Offline":[63],"Q-learning":[64,76,107,119],"into":[65,167],"LLM":[66],"models.":[68],"We":[69],"address":[70],"three":[71],"key":[72],"challenges":[73],"applying":[75],"to":[77,152,196,219],"LLMs:":[78],"(1)":[79],"handling":[80],"utterance-level":[81],"Markov":[82],"Decision":[83],"Processes":[84],"(MDPs),":[85],"(2)":[86],"managing":[87],"large":[88],"action":[89,111],"spaces,":[90],"and":[91,114,132,180,206,214],"(3)":[92],"mitigating":[93],"overestimation":[94],"bias.":[95],"VerifierQ":[96,147,217],"modified":[99],"Bellman":[100],"update":[101],"for":[102,109,122,145],"bounded":[103],"Q-values,":[104],"incorporates":[105],"Implicit":[106],"(IQL)":[108],"efficient":[110],"space":[112],"management,":[113],"Conservative":[118],"(CQL)":[120],"formulation":[121],"balanced":[123],"Q-value":[124,130],"estimation.":[125],"Our":[126],"method":[127],"enables":[128],"parallel":[129],"computation":[131],"improving":[133],"training":[134],"efficiency.":[135],"While":[136],"recent":[137],"work":[138],"has":[139],"explored":[140],"RL":[141,165],"techniques":[142],"like":[143],"MCTS":[144],"generators,":[146],"is":[148],"among":[149],"first":[151],"investigate":[153],"(critic)":[156],"aspect":[157],"integration":[163],"principles":[166],"complements":[170],"existing":[171],"generator":[174],"techniques,":[175],"potentially":[176],"enabling":[177],"more":[178],"robust":[179],"adaptive":[181],"LLMs.":[184],"Experimental":[185],"results":[186],"mathematical":[188],"tasks":[190,230],"demonstrate":[191],"VerifierQ's":[192],"superior":[193],"performance":[194],"compared":[195],"traditional":[197],"approaches,":[200],"with":[201],"improvements":[202],"efficiency,":[204],"accuracy":[205],"robustness.":[207],"By":[208],"enhancing":[209],"synergy":[211],"between":[212],"generation":[213],"evaluation":[215],"capabilities,":[216],"contributes":[218],"ongoing":[221],"evolution":[222],"AI":[224],"systems":[225],"addressing":[227],"complex":[228],"cognitive":[229],"across":[231],"various":[232],"domains.":[233]},"counts_by_year":[],"updated_date":"2026-08-23T07:36:19.812096","created_date":"2025-10-10T00:00:00"}