{"id":"https://openalex.org/W4367060663","doi":"https://doi.org/10.48550/arxiv.2304.12567","title":"Proto-Value Networks: Scaling Representation Learning with Auxiliary Tasks","display_name":"Proto-Value Networks: Scaling Representation Learning with Auxiliary Tasks","publication_year":2023,"publication_date":"2023-04-25","ids":{"openalex":"https://openalex.org/W4367060663","doi":"https://doi.org/10.48550/arxiv.2304.12567"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2304.12567","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2304.12567","pdf_url":"https://arxiv.org/pdf/2304.12567","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2304.12567","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5091650942","display_name":"Jesse Farebrother","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Farebrother, Jesse","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5075146281","display_name":"Joshua Greaves","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Greaves, Joshua","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5070953294","display_name":"Rishabh Agarwal","orcid":"https://orcid.org/0000-0003-2817-0231"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Agarwal, Rishabh","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5033033832","display_name":"Charline Le Lan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lan, Charline Le","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5029480918","display_name":"Ross Goroshin","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Goroshin, Ross","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5068291173","display_name":"Pablo Samuel Castro","orcid":"https://orcid.org/0000-0002-3206-336X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Castro, Pablo Samuel","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5001087292","display_name":"Marc G. Bellemare","orcid":"https://orcid.org/0000-0002-6096-0105"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Bellemare, Marc G.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9919999837875366,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9919999837875366,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.7756565809249878},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7535810470581055},{"id":"https://openalex.org/keywords/successor-cardinal","display_name":"Successor cardinal","score":0.7231788635253906},{"id":"https://openalex.org/keywords/representation","display_name":"Representation (politics)","score":0.6650395393371582},{"id":"https://openalex.org/keywords/value","display_name":"Value (mathematics)","score":0.6392455101013184},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5941725969314575},{"id":"https://openalex.org/keywords/function","display_name":"Function (biology)","score":0.5007164478302002},{"id":"https://openalex.org/keywords/carry","display_name":"Carry (investment)","score":0.47003716230392456},{"id":"https://openalex.org/keywords/feature-learning","display_name":"Feature learning","score":0.44664981961250305},{"id":"https://openalex.org/keywords/object","display_name":"Object (grammar)","score":0.41890907287597656},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.41322553157806396},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.14806342124938965}],"concepts":[{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.7756565809249878},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7535810470581055},{"id":"https://openalex.org/C75306776","wikidata":"https://www.wikidata.org/wiki/Q7632662","display_name":"Successor cardinal","level":2,"score":0.7231788635253906},{"id":"https://openalex.org/C2776359362","wikidata":"https://www.wikidata.org/wiki/Q2145286","display_name":"Representation (politics)","level":3,"score":0.6650395393371582},{"id":"https://openalex.org/C2776291640","wikidata":"https://www.wikidata.org/wiki/Q2912517","display_name":"Value (mathematics)","level":2,"score":0.6392455101013184},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5941725969314575},{"id":"https://openalex.org/C14036430","wikidata":"https://www.wikidata.org/wiki/Q3736076","display_name":"Function (biology)","level":2,"score":0.5007164478302002},{"id":"https://openalex.org/C2776299755","wikidata":"https://www.wikidata.org/wiki/Q432449","display_name":"Carry (investment)","level":2,"score":0.47003716230392456},{"id":"https://openalex.org/C59404180","wikidata":"https://www.wikidata.org/wiki/Q17013334","display_name":"Feature learning","level":2,"score":0.44664981961250305},{"id":"https://openalex.org/C2781238097","wikidata":"https://www.wikidata.org/wiki/Q175026","display_name":"Object (grammar)","level":2,"score":0.41890907287597656},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.41322553157806396},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.14806342124938965},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C78458016","wikidata":"https://www.wikidata.org/wiki/Q840400","display_name":"Evolutionary biology","level":1,"score":0.0},{"id":"https://openalex.org/C94625758","wikidata":"https://www.wikidata.org/wiki/Q7163","display_name":"Politics","level":2,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C10138342","wikidata":"https://www.wikidata.org/wiki/Q43015","display_name":"Finance","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2304.12567","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2304.12567","pdf_url":"https://arxiv.org/pdf/2304.12567","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},{"id":"doi:10.48550/arxiv.2304.12567","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2304.12567","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2304.12567","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2304.12567","pdf_url":"https://arxiv.org/pdf/2304.12567","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320309949","display_name":"Canadian Institute for Advanced Research","ror":"https://ror.org/01sdtdd95"},{"id":"https://openalex.org/F4320334593","display_name":"Natural Sciences and Engineering Research Council of Canada","ror":"https://ror.org/01h531d29"}],"has_content":{"grobid_xml":false,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4367060663.pdf"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W2473542051","https://openalex.org/W3123320268","https://openalex.org/W2994519609","https://openalex.org/W2726593096","https://openalex.org/W2054269721","https://openalex.org/W4235459050","https://openalex.org/W2767501615","https://openalex.org/W2390822611","https://openalex.org/W2908217413","https://openalex.org/W2158654672"],"abstract_inverted_index":{"Auxiliary":[0],"tasks":[1,48,77,90,111,118],"improve":[2],"the":[3,65,73,84,87,92,95,114,135,162,173,208],"representations":[4],"learned":[5],"by":[6],"deep":[7,155],"reinforcement":[8,156],"learning":[9,30,38,79,133,140,157],"agents.":[10],"Analytically,":[11],"their":[12,21],"effect":[13],"is":[14,41,137],"reasonably":[15],"well":[16],"understood;":[17],"in":[18,25],"practice,":[19],"however,":[20],"primary":[22],"use":[23],"remains":[24],"support":[26],"of":[27,62,75,89,94,109,170,205],"a":[28,35,106,130,138,168,201],"main":[29],"objective,":[31],"rather":[32],"than":[33],"as":[34,57,146],"method":[36],"for":[37,78],"representations.":[39],"This":[40],"perhaps":[42],"surprising":[43],"given":[44],"that":[45,142,179,185],"many":[46],"auxiliary":[47,76,110],"are":[49,98,119],"defined":[50],"procedurally,":[51],"and":[52,91,123,200],"hence":[53],"can":[54,143],"be":[55,144,187],"treated":[56],"an":[58],"essentially":[59],"infinite":[60],"source":[61],"information":[63],"about":[64],"environment.":[66],"Based":[67],"on":[68,83,113,172],"this":[69,102],"observation,":[70],"we":[71,104,160,177],"study":[72],"effectiveness":[74],"rich":[80,183],"representations,":[81],"focusing":[82],"setting":[85],"where":[86],"number":[88,203],"size":[93],"agent's":[96],"network":[97],"simultaneously":[99],"increased.":[100],"For":[101],"purpose,":[103],"derive":[105],"new":[107],"family":[108],"based":[112],"successor":[115],"measure.":[116],"These":[117],"easy":[120],"to":[121,154,189,193],"implement":[122],"have":[124],"appealing":[125],"theoretical":[126],"properties.":[127],"Combined":[128],"with":[129,207],"suitable":[131],"off-policy":[132],"rule,":[134],"result":[136],"representation":[139],"algorithm":[141],"understood":[145],"extending":[147],"Mahadevan":[148],"&":[149],"Maggioni":[150],"(2007)'s":[151],"proto-value":[152,165,180],"functions":[153],"--":[158],"accordingly,":[159],"call":[161],"resulting":[163],"object":[164],"networks.":[166],"Through":[167],"series":[169],"experiments":[171],"Arcade":[174],"Learning":[175],"Environment,":[176],"demonstrate":[178],"networks":[181],"produce":[182],"features":[184],"may":[186],"used":[188],"obtain":[190],"performance":[191],"comparable":[192],"established":[194],"algorithms,":[195],"using":[196],"only":[197],"linear":[198],"approximation":[199],"small":[202],"(~4M)":[204],"interactions":[206],"environment's":[209],"reward":[210],"function.":[211]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2023-04-27T00:00:00"}