{"id":"https://openalex.org/W4392121074","doi":"https://doi.org/10.48550/arxiv.2402.14688","title":"Q-Probe: A Lightweight Approach to Reward Maximization for Language Models","display_name":"Q-Probe: A Lightweight Approach to Reward Maximization for Language Models","publication_year":2024,"publication_date":"2024-02-22","ids":{"openalex":"https://openalex.org/W4392121074","doi":"https://doi.org/10.48550/arxiv.2402.14688"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2402.14688","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2402.14688","pdf_url":"https://arxiv.org/pdf/2402.14688","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2402.14688","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5102795202","display_name":"Kenneth Li","orcid":"https://orcid.org/0009-0007-9132-915X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Kenneth","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5048166612","display_name":"Samy Jelassi","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jelassi, Samy","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5047201942","display_name":"Hugh Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Hugh","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5108381794","display_name":"Sham M. Kakade","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kakade, Sham","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5039276358","display_name":"Martin Wattenberg","orcid":"https://orcid.org/0000-0003-0904-4862"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wattenberg, Martin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5037978510","display_name":"David Brandfonbrener","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Brandfonbrener, David","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9871000051498413,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9871000051498413,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.98580002784729,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/maximization","display_name":"Maximization","score":0.6311867833137512},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5451603531837463},{"id":"https://openalex.org/keywords/utility-maximization","display_name":"Utility maximization","score":0.41718778014183044},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.246160089969635},{"id":"https://openalex.org/keywords/economics","display_name":"Economics","score":0.18273025751113892},{"id":"https://openalex.org/keywords/mathematical-economics","display_name":"Mathematical economics","score":0.10301899909973145},{"id":"https://openalex.org/keywords/social-psychology","display_name":"Social psychology","score":0.08080711960792542}],"concepts":[{"id":"https://openalex.org/C2776330181","wikidata":"https://www.wikidata.org/wiki/Q18358244","display_name":"Maximization","level":2,"score":0.6311867833137512},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5451603531837463},{"id":"https://openalex.org/C2985793214","wikidata":"https://www.wikidata.org/wiki/Q3274096","display_name":"Utility maximization","level":2,"score":0.41718778014183044},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.246160089969635},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.18273025751113892},{"id":"https://openalex.org/C144237770","wikidata":"https://www.wikidata.org/wiki/Q747534","display_name":"Mathematical economics","level":1,"score":0.10301899909973145},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.08080711960792542}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2402.14688","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2402.14688","pdf_url":"https://arxiv.org/pdf/2402.14688","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2402.14688","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2402.14688","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2402.14688","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2402.14688","pdf_url":"https://arxiv.org/pdf/2402.14688","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G4720003262","display_name":null,"funder_award_id":"N00014-22","funder_id":"https://openalex.org/F4320337345","funder_display_name":"Office of Naval Research"},{"id":"https://openalex.org/G8876996369","display_name":null,"funder_award_id":"N00014","funder_id":"https://openalex.org/F4320337345","funder_display_name":"Office of Naval Research"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320309622","display_name":"Harvard University","ror":"https://ror.org/03vek6s52"},{"id":"https://openalex.org/F4320337345","display_name":"Office of Naval Research","ror":"https://ror.org/00rk2pe57"},{"id":"https://openalex.org/F4320337380","display_name":"Division of Mathematical Sciences","ror":"https://ror.org/051fftw81"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4392121074.pdf","grobid_xml":"https://content.openalex.org/works/W4392121074.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W3121521771","https://openalex.org/W30823005","https://openalex.org/W2393902186","https://openalex.org/W4388998647","https://openalex.org/W2267645079","https://openalex.org/W2741533395","https://openalex.org/W4296628518","https://openalex.org/W4380149910","https://openalex.org/W2490250203","https://openalex.org/W2118922860"],"abstract_inverted_index":{"We":[0,67],"present":[1],"an":[2,150],"approach":[3],"called":[4],"Q-probing":[5,22],"to":[6,12,48,63,76,157],"adapt":[7],"a":[8,14,19,50,55,77,99,142],"pre-trained":[9],"language":[10],"model":[11],"maximize":[13],"task-specific":[15],"reward":[16,96],"function.":[17],"At":[18],"high":[20],"level,":[21],"sits":[23],"between":[24],"heavier":[25],"approaches":[26,32],"such":[27,33],"as":[28,34,83,126,128],"finetuning":[29,137],"and":[30,159],"lighter":[31],"few":[35],"shot":[36],"prompting,":[37],"but":[38],"can":[39,60,144],"also":[40],"be":[41,61,145],"combined":[42],"with":[43,121],"either.":[44],"The":[45],"idea":[46],"is":[47,74],"learn":[49],"simple":[51],"linear":[52],"function":[53],"on":[54,108,147],"model's":[56],"embedding":[57],"space":[58],"that":[59,70],"used":[62],"reweight":[64],"candidate":[65],"completions.":[66],"theoretically":[68],"show":[69],"this":[71,114],"sampling":[72,158],"procedure":[73],"equivalent":[75],"KL-constrained":[78],"maximization":[79],"of":[80,86,101,149],"the":[81,84,91],"Q-probe":[82,143],"number":[85],"samples":[87],"increases.":[88],"To":[89],"train":[90],"Q-probes":[92],"we":[93,116],"consider":[94],"either":[95],"modeling":[97],"or":[98],"class":[100],"novel":[102],"direct":[103],"policy":[104,111],"learning":[105],"objectives":[106],"based":[107],"importance":[109],"weighted":[110],"gradients.":[112],"With":[113],"technique,":[115],"see":[117],"gains":[118],"in":[119,138],"domains":[120],"ground-truth":[122],"rewards":[123,130],"(code":[124],"generation)":[125],"well":[127],"implicit":[129],"defined":[131],"by":[132],"preference":[133],"data,":[134],"even":[135],"outperforming":[136],"data-limited":[139],"regimes.":[140],"Moreover,":[141],"trained":[146],"top":[148],"API":[151],"since":[152],"it":[153],"only":[154],"assumes":[155],"access":[156],"embeddings.":[160],"Code:":[161],"https://github.com/likenneth/q_probe":[162],".":[163]},"counts_by_year":[],"updated_date":"2026-08-26T07:47:46.906454","created_date":"2024-02-24T00:00:00"}