{"id":"https://openalex.org/W4407764293","doi":"https://doi.org/10.48550/arxiv.2502.13417","title":"RLTHF: Targeted Human Feedback for LLM Alignment","display_name":"RLTHF: Targeted Human Feedback for LLM Alignment","publication_year":2025,"publication_date":"2025-02-19","ids":{"openalex":"https://openalex.org/W4407764293","doi":"https://doi.org/10.48550/arxiv.2502.13417"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2502.13417","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2502.13417","pdf_url":"https://arxiv.org/pdf/2502.13417","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2502.13417","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5069256680","display_name":"Yifei Xu","orcid":"https://orcid.org/0000-0003-1329-3124","observed_orcids":["https://orcid.org/0000-0003-1329-3124"]},"institutions":[{"id":"https://openalex.org/I161318765","display_name":"University of California, Los Angeles","ror":"https://ror.org/046rm7j60","country_code":"US","type":"education","lineage":["https://openalex.org/I161318765"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Xu, Yifei","raw_affiliation_strings":["University of California , Los Angeles"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of California , Los Angeles","institution_ids":["https://openalex.org/I161318765"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5043473455","display_name":"Tusher Chakraborty","orcid":"https://orcid.org/0000-0003-1656-5471","observed_orcids":["https://orcid.org/0000-0003-1656-5471"]},"institutions":[{"id":"https://openalex.org/I161318765","display_name":"University of California, Los Angeles","ror":"https://ror.org/046rm7j60","country_code":"US","type":"education","lineage":["https://openalex.org/I161318765"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Chakraborty, Tusher","raw_affiliation_strings":["University of California , Los Angeles"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of California , Los Angeles","institution_ids":["https://openalex.org/I161318765"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5079458476","display_name":"Emre K\u0131c\u0131man","orcid":"https://orcid.org/0000-0001-5429-468X","observed_orcids":["https://orcid.org/0000-0001-5429-468X"]},"institutions":[{"id":"https://openalex.org/I161318765","display_name":"University of California, Los Angeles","ror":"https://ror.org/046rm7j60","country_code":"US","type":"education","lineage":["https://openalex.org/I161318765"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"K\u0131c\u0131man, Emre","raw_affiliation_strings":["University of California , Los Angeles"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of California , Los Angeles","institution_ids":["https://openalex.org/I161318765"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Aryal, Bibek","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I161318765","display_name":"University of California, Los Angeles","ror":"https://ror.org/046rm7j60","country_code":"US","type":"education","lineage":["https://openalex.org/I161318765"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Aryal, Bibek","raw_affiliation_strings":["University of California , Los Angeles"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of California , Los Angeles","institution_ids":["https://openalex.org/I161318765"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Rodrigues, Eduardo","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I161318765","display_name":"University of California, Los Angeles","ror":"https://ror.org/046rm7j60","country_code":"US","type":"education","lineage":["https://openalex.org/I161318765"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Rodrigues, Eduardo","raw_affiliation_strings":["University of California , Los Angeles"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of California , Los Angeles","institution_ids":["https://openalex.org/I161318765"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103899562","display_name":"Srinagesh Sharma","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I161318765","display_name":"University of California, Los Angeles","ror":"https://ror.org/046rm7j60","country_code":"US","type":"education","lineage":["https://openalex.org/I161318765"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Sharma, Srinagesh","raw_affiliation_strings":["University of California , Los Angeles"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of California , Los Angeles","institution_ids":["https://openalex.org/I161318765"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Estevao, Roberto","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I161318765","display_name":"University of California, Los Angeles","ror":"https://ror.org/046rm7j60","country_code":"US","type":"education","lineage":["https://openalex.org/I161318765"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Estevao, Roberto","raw_affiliation_strings":["University of California , Los Angeles"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of California , Los Angeles","institution_ids":["https://openalex.org/I161318765"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5012552344","display_name":"Mar\u00eda Angels de Luis Balaguer","orcid":"https://orcid.org/0009-0004-1687-2716","observed_orcids":["https://orcid.org/0009-0004-1687-2716","https://orcid.org/0000-0002-6272-7841"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Balaguer, Maria Angels de Luis","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Wolk, Jessica","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I161318765","display_name":"University of California, Los Angeles","ror":"https://ror.org/046rm7j60","country_code":"US","type":"education","lineage":["https://openalex.org/I161318765"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Wolk, Jessica","raw_affiliation_strings":["University of California , Los Angeles"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of California , Los Angeles","institution_ids":["https://openalex.org/I161318765"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5027978607","display_name":"Rafael Padilha","orcid":"https://orcid.org/0000-0003-1944-5475","observed_orcids":["https://orcid.org/0000-0003-1944-5475"]},"institutions":[{"id":"https://openalex.org/I161318765","display_name":"University of California, Los Angeles","ror":"https://ror.org/046rm7j60","country_code":"US","type":"education","lineage":["https://openalex.org/I161318765"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Padilha, Rafael","raw_affiliation_strings":["University of California , Los Angeles"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of California , Los Angeles","institution_ids":["https://openalex.org/I161318765"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5033463791","display_name":"Leonardo Silva Nunes","orcid":"https://orcid.org/0009-0009-1296-1013","observed_orcids":["https://orcid.org/0009-0009-1296-1013","https://orcid.org/0000-0001-8489-7043"]},"institutions":[{"id":"https://openalex.org/I161318765","display_name":"University of California, Los Angeles","ror":"https://ror.org/046rm7j60","country_code":"US","type":"education","lineage":["https://openalex.org/I161318765"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Nunes, Leonardo","raw_affiliation_strings":["University of California , Los Angeles"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of California , Los Angeles","institution_ids":["https://openalex.org/I161318765"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5110322584","display_name":"Shobana Balakrishnan","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I161318765","display_name":"University of California, Los Angeles","ror":"https://ror.org/046rm7j60","country_code":"US","type":"education","lineage":["https://openalex.org/I161318765"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Balakrishnan, Shobana","raw_affiliation_strings":["University of California , Los Angeles"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of California , Los Angeles","institution_ids":["https://openalex.org/I161318765"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5020188879","display_name":"Songwu Lu","orcid":"https://orcid.org/0000-0003-3779-0918","observed_orcids":["https://orcid.org/0000-0003-3779-0918"]},"institutions":[{"id":"https://openalex.org/I161318765","display_name":"University of California, Los Angeles","ror":"https://ror.org/046rm7j60","country_code":"US","type":"education","lineage":["https://openalex.org/I161318765"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Lu, Songwu","raw_affiliation_strings":["University of California , Los Angeles"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of California , Los Angeles","institution_ids":["https://openalex.org/I161318765"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5112443217","display_name":"Ranveer Chandra","orcid":"https://orcid.org/0000-0002-4175-1404","observed_orcids":["https://orcid.org/0000-0002-4175-1404"]},"institutions":[{"id":"https://openalex.org/I161318765","display_name":"University of California, Los Angeles","ror":"https://ror.org/046rm7j60","country_code":"US","type":"education","lineage":["https://openalex.org/I161318765"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Chandra, Ranveer","raw_affiliation_strings":["University of California , Los Angeles"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of California , Los Angeles","institution_ids":["https://openalex.org/I161318765"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":15,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.3490999937057495,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.3490999937057495,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.1282999962568283,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.10559999942779541,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/large-language-model-alignment","display_name":"large language model alignment","score":0.7570000290870667},{"id":"https://openalex.org/keywords/reinforcement-learning-from-human-feedback","display_name":"reinforcement learning from human feedback","score":0.9309999942779541},{"id":"https://openalex.org/keywords/annotation-cost-reduction","display_name":"annotation cost reduction","score":0.5210000276565552}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4509413540363312}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2502.13417","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2502.13417","pdf_url":"https://arxiv.org/pdf/2502.13417","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2502.13417","doi":"https://doi.org/10.48550/arxiv.2502.13417","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2502.13417","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2502.13417","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2502.13417","pdf_url":"https://arxiv.org/pdf/2502.13417","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4407764293.pdf","grobid_xml":"https://content.openalex.org/works/W4407764293.grobid-xml"},"referenced_works_count":26,"referenced_works":["https://openalex.org/W2736601468","https://openalex.org/W3198002980","https://openalex.org/W4223908421","https://openalex.org/W4226399820","https://openalex.org/W4311991106","https://openalex.org/W4322718191","https://openalex.org/W4327810158","https://openalex.org/W4384111857","https://openalex.org/W4385848337","https://openalex.org/W4387355948","https://openalex.org/W4390041933","https://openalex.org/W4390723197","https://openalex.org/W4390833718","https://openalex.org/W4391047180","https://openalex.org/W4391555989","https://openalex.org/W4394906585","https://openalex.org/W4401306886","https://openalex.org/W4403753192","https://openalex.org/W4403781093","https://openalex.org/W4404344935","https://openalex.org/W4404360748","https://openalex.org/W4405627094","https://openalex.org/W4406492615","https://openalex.org/W4415797148","https://openalex.org/W4415798425","https://openalex.org/W7166879155"],"related_works":["https://openalex.org/W7131779452","https://openalex.org/W4412587432","https://openalex.org/W7170881200","https://openalex.org/W4376167364","https://openalex.org/W4365211596","https://openalex.org/W4393729321","https://openalex.org/W4402683794","https://openalex.org/W4389114181","https://openalex.org/W7162487684","https://openalex.org/W7149004367"],"abstract_inverted_index":{"Fine-tuning":[0],"large":[1],"language":[2],"models":[3,113],"(LLMs)":[4],"to":[5,13,55],"align":[6],"with":[7,51,60,104],"user":[8],"preferences":[9],"is":[10],"challenging":[11],"due":[12],"the":[14,29,108,130],"high":[15],"cost":[16],"of":[17,32,107,132],"quality":[18],"human":[19,53,83,109],"annotations":[20,54],"in":[21],"Reinforcement":[22],"Learning":[23],"from":[24],"Human":[25],"Feedback":[26],"(RLHF)":[27],"and":[28,76,94],"generalizability":[30],"limitations":[31],"AI":[33],"Feedback.":[34],"To":[35],"address":[36],"these":[37],"challenges,":[38],"we":[39],"propose":[40],"RLTHF,":[41],"a":[42,71],"human-AI":[43],"hybrid":[44],"framework":[45],"that":[46,98],"combines":[47],"LLM-based":[48],"initial":[49],"alignment":[50,59,79,103],"selective":[52],"achieve":[56],"full-human":[57,101],"annotation":[58,110],"minimal":[61],"effort.":[62,111],"RLTHF":[63,99],"identifies":[64],"hard-to-annotate":[65],"samples":[66],"mislabeled":[67],"by":[68,80],"LLMs":[69],"using":[70],"reward":[72,74],"model's":[73],"distribution":[75],"iteratively":[77],"enhances":[78],"integrating":[81],"strategic":[82],"corrections":[84],"while":[85],"leveraging":[86],"LLM's":[87],"correctly":[88],"labeled":[89],"samples.":[90],"Evaluations":[91],"on":[92,115,125],"HH-RLHF":[93],"TL;DR":[95],"datasets":[96,118],"show":[97],"reaches":[100],"annotation-level":[102],"only":[105],"6-7%":[106],"Furthermore,":[112],"trained":[114,124],"RLTHF's":[116],"curated":[117],"for":[119],"downstream":[120],"tasks":[121],"outperform":[122],"those":[123],"fully":[126],"human-annotated":[127],"datasets,":[128],"underscoring":[129],"effectiveness":[131],"RLTHF.":[133]},"counts_by_year":[{"year":2026,"cited_by_count":9},{"year":2025,"cited_by_count":5},{"year":2024,"cited_by_count":1}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2025-02-21T00:00:00"}