{"id":"https://openalex.org/W4292218840","doi":"https://doi.org/10.48550/arxiv.2208.07563","title":"Reinforcement Learning to Rank Using Coarse-grained Rewards","display_name":"Reinforcement Learning to Rank Using Coarse-grained Rewards","publication_year":2022,"publication_date":"2022-08-16","ids":{"openalex":"https://openalex.org/W4292218840","doi":"https://doi.org/10.48550/arxiv.2208.07563"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2208.07563","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2208.07563","pdf_url":"https://arxiv.org/pdf/2208.07563","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2208.07563","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Tu, Yiteng","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Tu, Yiteng","raw_affiliation_strings":["DCST , Tsinghua University Beijing , China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"DCST , Tsinghua University Beijing , China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5036683049","display_name":"Zhichao Xu","orcid":"https://orcid.org/0000-0002-2370-4487","observed_orcids":["https://orcid.org/0000-0002-2370-4487","https://orcid.org/0009-0002-2475-2031"]},"institutions":[{"id":"https://openalex.org/I223532165","display_name":"University of Utah","ror":"https://ror.org/03r0ha626","country_code":"US","type":"education","lineage":["https://openalex.org/I223532165"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Xu, Zhichao","raw_affiliation_strings":["University of Utah Salt Lake City , United States"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Utah Salt Lake City , United States","institution_ids":["https://openalex.org/I223532165"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100338214","display_name":"Tao Yang","orcid":"https://orcid.org/0000-0002-7282-2463","observed_orcids":["https://orcid.org/0000-0002-7282-2463"]},"institutions":[{"id":"https://openalex.org/I1311688040","display_name":"Amazon (United States)","ror":"https://ror.org/04mv4n011","country_code":"US","type":"company","lineage":["https://openalex.org/I1311688040"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yang, Tao","raw_affiliation_strings":["Amazon.com Inc Palo Alto , United States"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Amazon.com Inc Palo Alto , United States","institution_ids":["https://openalex.org/I1311688040"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Su, Weihang","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Su, Weihang","raw_affiliation_strings":["DCST , Tsinghua University Beijing , China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"DCST , Tsinghua University Beijing , China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Zhou, Yujia","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhou, Yujia","raw_affiliation_strings":["DCST , Tsinghua University Beijing , China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"DCST , Tsinghua University Beijing , China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Liu, Yiqun","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Liu, Yiqun","raw_affiliation_strings":["DCST , Tsinghua University Beijing , China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"DCST , Tsinghua University Beijing , China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Lin, Fen","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I2250653659","display_name":"Tencent (China)","ror":"https://ror.org/00hhjss72","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250653659"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Lin, Fen","raw_affiliation_strings":["Tencent Beijing , China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tencent Beijing , China","institution_ids":["https://openalex.org/I2250653659"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Liu, Qin","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I2250653659","display_name":"Tencent (China)","ror":"https://ror.org/00hhjss72","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250653659"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Liu, Qin","raw_affiliation_strings":["Tencent Beijing , China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tencent Beijing , China","institution_ids":["https://openalex.org/I2250653659"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5089655391","display_name":"Qingyao Ai","orcid":"https://orcid.org/0000-0002-5030-709X","observed_orcids":["https://orcid.org/0000-0002-5030-709X"]},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Ai, Qingyao","raw_affiliation_strings":["DCST , Tsinghua University Beijing , China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"DCST , Tsinghua University Beijing , China","institution_ids":["https://openalex.org/I99065089"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":7,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9984999895095825,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9984999895095825,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10203","display_name":"Recommender Systems and Techniques","score":0.0008999999845400453,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":9.999999747378752e-05,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/learning-to-rank","display_name":"learning to rank","score":0.902999997138977},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.7789999842643738},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"information retrieval","score":0.5889999866485596},{"id":"https://openalex.org/keywords/sample-efficiency","display_name":"sample efficiency","score":0.6480000019073486},{"id":"https://openalex.org/keywords/reward-variance","display_name":"reward variance","score":0.6230000257492065}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8321652412414551},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.7492615580558777},{"id":"https://openalex.org/C86037889","wikidata":"https://www.wikidata.org/wiki/Q4330127","display_name":"Learning to rank","level":3,"score":0.68268221616745},{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.6801724433898926},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5976044535636902},{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.5840204954147339},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5627421140670776},{"id":"https://openalex.org/C164226766","wikidata":"https://www.wikidata.org/wiki/Q7293202","display_name":"Rank (graph theory)","level":2,"score":0.5583401918411255},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.5114539861679077},{"id":"https://openalex.org/C26713055","wikidata":"https://www.wikidata.org/wiki/Q245962","display_name":"Implementation","level":2,"score":0.4744371473789215},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.46793076395988464},{"id":"https://openalex.org/C2984842247","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep neural networks","level":3,"score":0.419140100479126},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.41654980182647705},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C114614502","wikidata":"https://www.wikidata.org/wiki/Q76592","display_name":"Combinatorics","level":1,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2208.07563","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2208.07563","pdf_url":"https://arxiv.org/pdf/2208.07563","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2208.07563","doi":"https://doi.org/10.48550/arxiv.2208.07563","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2208.07563","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2208.07563","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2208.07563","pdf_url":"https://arxiv.org/pdf/2208.07563","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4292218840.pdf","grobid_xml":"https://content.openalex.org/works/W4292218840.grobid-xml"},"referenced_works_count":10,"referenced_works":["https://openalex.org/W1530210183","https://openalex.org/W2736601468","https://openalex.org/W2908510526","https://openalex.org/W2964199361","https://openalex.org/W2996726407","https://openalex.org/W4327810158","https://openalex.org/W4391631327","https://openalex.org/W4404330874","https://openalex.org/W4414281281","https://openalex.org/W4414939355"],"related_works":["https://openalex.org/W4391549777","https://openalex.org/W7108362331","https://openalex.org/W7168283114","https://openalex.org/W4416196420","https://openalex.org/W6891997121","https://openalex.org/W6979303653","https://openalex.org/W7105507089","https://openalex.org/W7126015098","https://openalex.org/W2918618374","https://openalex.org/W7167913655"],"abstract_inverted_index":{"Learning":[0,63,81],"to":[1,69,82],"rank":[2],"(LTR)":[3],"plays":[4],"a":[5,66],"crucial":[6],"role":[7],"in":[8,98],"various":[9,134],"Information":[10],"Retrieval":[11],"(IR)":[12],"tasks.":[13],"Although":[14],"supervised":[15,128,161],"LTR":[16,143],"methods":[17,116,132,151],"based":[18,117],"on":[19,32,118,141],"fine-grained":[20,166],"relevance":[21],"labels":[22],"(e.g.,":[23],"document-level":[24],"annotations)":[25],"have":[26],"achieved":[27],"significant":[28],"success,":[29],"their":[30],"reliance":[31],"costly":[33],"and":[34,40,54,60,90,112,130,137,158],"potentially":[35],"biased":[36],"annotations":[37],"limits":[38],"scalability":[39],"alignment":[41],"with":[42,109,165],"realistic":[43],"goals.":[44],"In":[45],"contrast,":[46],"coarse-grained":[47,110,138,156],"feedback":[48],"signals,":[49,76],"such":[50],"as":[51],"duration":[52],"time":[53],"session-level":[55],"engagement,":[56],"are":[57],"more":[58],"accessible":[59],"affordable.":[61],"Reinforcement":[62,80],"(RL)":[64],"offers":[65],"promising":[67],"framework":[68],"directly":[70,153],"optimize":[71],"these":[72],"objectives":[73],"using":[74],"reward":[75,139],"but":[77],"most":[78],"existing":[79],"Rank":[83],"(RLTR)":[84],"approaches":[85],"suffer":[86],"from":[87,155],"high":[88],"variance":[89],"low":[91],"sample":[92],"efficiency.":[93],"Motivated":[94],"by":[95],"recent":[96],"advances":[97],"large":[99],"language":[100],"models":[101],"(LLMs),":[102],"we":[103],"re-examine":[104],"the":[105,170],"problem":[106],"of":[107,173],"RLTR":[108,115,174],"rewards":[111,157],"propose":[113],"new":[114],"widely":[119],"used":[120],"RL":[121,150],"algorithms":[122],"for":[123,175],"LLMs.":[124],"We":[125],"systematically":[126],"compare":[127],"learning":[129,162],"RL-based":[131],"across":[133],"model":[135],"architectures":[136],"functions":[140],"large-scale":[142],"benchmarks.":[144],"Experimental":[145],"results":[146],"demonstrate":[147],"that":[148],"advanced":[149],"can":[152],"learn":[154],"outperform":[159],"strong":[160],"baselines":[163],"even":[164],"labels.":[167],"This":[168],"shows":[169],"great":[171],"potential":[172],"metric-agnostic":[176],"ranking":[177],"optimization.":[178]},"counts_by_year":[{"year":2025,"cited_by_count":4},{"year":2024,"cited_by_count":1},{"year":2023,"cited_by_count":2}],"updated_date":"2026-11-10T11:39:43.021295","created_date":"2022-08-18T00:00:00"}