{"id":"https://openalex.org/W7137865038","doi":"https://doi.org/10.1609/aaai.v40i21.38806","title":"Mitigating Length Bias in RLHF Through a Causal Lens","display_name":"Mitigating Length Bias in RLHF Through a Causal Lens","publication_year":2026,"publication_date":"2026-03-14","ids":{"openalex":"https://openalex.org/W7137865038","doi":"https://doi.org/10.1609/aaai.v40i21.38806"},"language":null,"primary_location":{"id":"doi:10.1609/aaai.v40i21.38806","is_oa":true,"landing_page_url":"https://doi.org/10.1609/aaai.v40i21.38806","pdf_url":null,"source":{"id":"https://openalex.org/S4210191458","display_name":"Proceedings of the AAAI Conference on Artificial Intelligence","issn_l":"2159-5399","issn":["2159-5399","2374-3468"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/P4310320058","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/P4310320058"],"host_organization_lineage_names":["Association for the Advancement of Artificial Intelligence"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Artificial Intelligence","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"diamond","oa_url":"https://doi.org/10.1609/aaai.v40i21.38806","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5001134221","display_name":"H.Y. Kim","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hyeonji Kim","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5129737812","display_name":"Sujeong Oh","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sujeong Oh","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5014657402","display_name":"Sanghack Lee","orcid":"https://orcid.org/0000-0001-7137-6126"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sanghack Lee","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.08167142,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"40","issue":"21","first_page":"17517","last_page":"17525"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.45410001277923584,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.45410001277923584,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.11429999768733978,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.039400000125169754,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/counterfactual-thinking","display_name":"Counterfactual thinking","score":0.8495000004768372},{"id":"https://openalex.org/keywords/robustness","display_name":"Robustness (evolution)","score":0.6603999733924866},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.47749999165534973},{"id":"https://openalex.org/keywords/construct","display_name":"Construct (python library)","score":0.39259999990463257},{"id":"https://openalex.org/keywords/causal-model","display_name":"Causal model","score":0.34950000047683716},{"id":"https://openalex.org/keywords/sensitivity","display_name":"Sensitivity (control systems)","score":0.335099995136261},{"id":"https://openalex.org/keywords/counterfactual-conditional","display_name":"Counterfactual conditional","score":0.33169999718666077}],"concepts":[{"id":"https://openalex.org/C108650721","wikidata":"https://www.wikidata.org/wiki/Q1783253","display_name":"Counterfactual thinking","level":2,"score":0.8495000004768372},{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.6603999733924866},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5633000135421753},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.47749999165534973},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.4526999890804291},{"id":"https://openalex.org/C2780801425","wikidata":"https://www.wikidata.org/wiki/Q5164392","display_name":"Construct (python library)","level":2,"score":0.39259999990463257},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.36039999127388},{"id":"https://openalex.org/C11671645","wikidata":"https://www.wikidata.org/wiki/Q5054567","display_name":"Causal model","level":2,"score":0.34950000047683716},{"id":"https://openalex.org/C21200559","wikidata":"https://www.wikidata.org/wiki/Q7451068","display_name":"Sensitivity (control systems)","level":2,"score":0.335099995136261},{"id":"https://openalex.org/C71889745","wikidata":"https://www.wikidata.org/wiki/Q1783264","display_name":"Counterfactual conditional","level":3,"score":0.33169999718666077},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.32010000944137573},{"id":"https://openalex.org/C2778152352","wikidata":"https://www.wikidata.org/wiki/Q5165061","display_name":"Content (measure theory)","level":2,"score":0.3116999864578247},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3073999881744385},{"id":"https://openalex.org/C120936955","wikidata":"https://www.wikidata.org/wiki/Q2155640","display_name":"Empirical research","level":2,"score":0.29910001158714294},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.29159998893737793},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.2831999957561493},{"id":"https://openalex.org/C43091099","wikidata":"https://www.wikidata.org/wiki/Q1067788","display_name":"Through-the-lens metering","level":3,"score":0.28189998865127563},{"id":"https://openalex.org/C6571938","wikidata":"https://www.wikidata.org/wiki/Q3274486","display_name":"Omitted-variable bias","level":2,"score":0.27720001339912415},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.267300009727478},{"id":"https://openalex.org/C99498987","wikidata":"https://www.wikidata.org/wiki/Q2210247","display_name":"Noise (video)","level":3,"score":0.2669999897480011},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.26429998874664307},{"id":"https://openalex.org/C2780586882","wikidata":"https://www.wikidata.org/wiki/Q7520643","display_name":"Simple (philosophy)","level":2,"score":0.258899986743927}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1609/aaai.v40i21.38806","is_oa":true,"landing_page_url":"https://doi.org/10.1609/aaai.v40i21.38806","pdf_url":null,"source":{"id":"https://openalex.org/S4210191458","display_name":"Proceedings of the AAAI Conference on Artificial Intelligence","issn_l":"2159-5399","issn":["2159-5399","2374-3468"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/P4310320058","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/P4310320058"],"host_organization_lineage_names":["Association for the Advancement of Artificial Intelligence"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Artificial Intelligence","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1609/aaai.v40i21.38806","is_oa":true,"landing_page_url":"https://doi.org/10.1609/aaai.v40i21.38806","pdf_url":null,"source":{"id":"https://openalex.org/S4210191458","display_name":"Proceedings of the AAAI Conference on Artificial Intelligence","issn_l":"2159-5399","issn":["2159-5399","2374-3468"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/P4310320058","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/P4310320058"],"host_organization_lineage_names":["Association for the Advancement of Artificial Intelligence"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Artificial Intelligence","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Reinforcement":[0],"learning":[1],"from":[2,71,131],"human":[3,16],"feedback":[4],"(RLHF)":[5],"is":[6,56],"widely":[7],"used":[8,78],"to":[9,28,53,67,79,86,126],"align":[10],"large":[11],"language":[12],"models":[13,21],"(LLMs)":[14],"with":[15,35,102],"preferences.":[17],"However,":[18],"RLHF-trained":[19],"reward":[20,50,82,122,154],"often":[22],"exhibit":[23],"length":[24,46,119,144],"bias\u2014a":[25],"systematic":[26],"tendency":[27],"favor":[29],"longer":[30],"responses":[31,88],"by":[32],"conflating":[33],"verbosity":[34],"quality.":[36],"We":[37],"propose":[38],"a":[39,57],"causal":[40],"framework":[41],"for":[42],"analyzing":[43],"and":[44,105,124,146,150],"mitigating":[45],"bias":[47,120,145],"in":[48,121,156],"RLHF":[49,157],"modeling.":[51],"Central":[52],"our":[54,116],"approach":[55,141],"counterfactual":[58,74],"data":[59],"augmentation":[60],"method":[61,117],"that":[62,115,138],"generates":[63],"response":[64],"pairs":[65,101,108],"designed":[66],"isolate":[68],"content":[69,91,104,151],"quality":[70,92],"verbosity.":[72,95],"These":[73,135],"examples":[75],"are":[76],"then":[77],"train":[80],"the":[81,132,139,148],"model,":[83],"enabling":[84],"it":[85],"assess":[87],"based":[89],"on":[90],"independently":[93],"of":[94,109,153],"Specifically,":[96],"we":[97],"construct":[98],"(1)":[99],"length-divergent":[100],"similar":[103,110],"(2)":[106],"content-divergent":[107],"length.":[111],"Empirical":[112],"evaluations":[113],"show":[114],"reduces":[118,143],"assignment":[123],"leads":[125],"more":[127],"concise,":[128],"content-focused":[129],"outputs":[130],"policy":[133],"model.":[134],"findings":[136],"demonstrate":[137],"proposed":[140],"effectively":[142],"improves":[147],"robustness":[149],"sensitivity":[152],"modeling":[155],"pipelines.":[158]},"counts_by_year":[],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2026-03-18T00:00:00"}