{"id":"https://openalex.org/W7202280720","doi":"https://doi.org/10.48550/arxiv.2608.10126","title":"Procedural Fairness Failures in RLHF from Preference Averaging","display_name":"Procedural Fairness Failures in RLHF from Preference Averaging","publication_year":2026,"publication_date":"2026-08-10","ids":{"openalex":"https://openalex.org/W7202280720","doi":"https://doi.org/10.48550/arxiv.2608.10126"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2608.10126","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2608.10126","pdf_url":"https://arxiv.org/pdf/2608.10126","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2608.10126","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5123827551","display_name":"M P V S Gopinadh","orcid":"https://orcid.org/0009-0000-9352-488X","observed_orcids":["https://orcid.org/0009-0000-9352-488X"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gopinadh, M P V S","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5146998782","display_name":"Karthik Kamuju","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kamuju, Karthik","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5146985574","display_name":"Kummari Avinash","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Avinash, Kummari","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136013448","display_name":"John Joshua","orcid":"https://orcid.org/0009-0006-4473-7751","observed_orcids":["https://orcid.org/0009-0006-4473-7751"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Joshua, John","raw_affiliation_strings":["Institute of Technology"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Technology","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5066515375","display_name":"Srinivasa Raju Rudraraju","orcid":"https://orcid.org/0000-0001-8826-8308","observed_orcids":["https://orcid.org/0000-0001-8826-8308"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Rudraraju, Srinivasa Raju","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.9761000275611877,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.9761000275611877,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.01269999984651804,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.0012000000569969416,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/procedural-justice","display_name":"procedural justice","score":0.800000011920929},{"id":"https://openalex.org/keywords/reinforcement-learning-from-human-feedback","display_name":"reinforcement learning from human feedback","score":0.8659999966621399},{"id":"https://openalex.org/keywords/preference-heterogeneity","display_name":"preference heterogeneity","score":0.7409999966621399},{"id":"https://openalex.org/keywords/reward-modeling","display_name":"reward modeling","score":0.7480000257492065},{"id":"https://openalex.org/keywords/ai-alignment","display_name":"AI alignment","score":0.5429999828338623},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.828000009059906},{"id":"https://openalex.org/keywords/agentic-systems","display_name":"agentic systems","score":0.6349999904632568}],"concepts":[{"id":"https://openalex.org/C2781249084","wikidata":"https://www.wikidata.org/wiki/Q908656","display_name":"Preference","level":2,"score":0.8291000127792358},{"id":"https://openalex.org/C181204326","wikidata":"https://www.wikidata.org/wiki/Q7239820","display_name":"Preference learning","level":3,"score":0.7409999966621399},{"id":"https://openalex.org/C194959371","wikidata":"https://www.wikidata.org/wiki/Q17056235","display_name":"Inequity aversion","level":3,"score":0.5029000043869019},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4941999912261963},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.4246000051498413},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.4072999954223633},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.39329999685287476},{"id":"https://openalex.org/C2779110102","wikidata":"https://www.wikidata.org/wiki/Q1323737","display_name":"Revealed preference","level":2,"score":0.39079999923706055},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.3684000074863434},{"id":"https://openalex.org/C67203356","wikidata":"https://www.wikidata.org/wiki/Q1321905","display_name":"Reinforcement","level":2,"score":0.36730000376701355},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.34860000014305115},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.3375000059604645},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.33649998903274536},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.32420000433921814},{"id":"https://openalex.org/C175444787","wikidata":"https://www.wikidata.org/wiki/Q39072","display_name":"Microeconomics","level":1,"score":0.31839999556541443},{"id":"https://openalex.org/C143661069","wikidata":"https://www.wikidata.org/wiki/Q670713","display_name":"Reward system","level":2,"score":0.3149000108242035},{"id":"https://openalex.org/C52820260","wikidata":"https://www.wikidata.org/wiki/Q7551312","display_name":"Social preferences","level":2,"score":0.3100000023841858},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.27469998598098755},{"id":"https://openalex.org/C2781043087","wikidata":"https://www.wikidata.org/wiki/Q939761","display_name":"Preference theory","level":3,"score":0.27149999141693115},{"id":"https://openalex.org/C2779149496","wikidata":"https://www.wikidata.org/wiki/Q442100","display_name":"Procedural justice","level":3,"score":0.2653999924659729},{"id":"https://openalex.org/C32172795","wikidata":"https://www.wikidata.org/wiki/Q4692266","display_name":"Aggregation problem","level":2,"score":0.25850000977516174}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2608.10126","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2608.10126","pdf_url":"https://arxiv.org/pdf/2608.10126","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2608.10126","doi":"https://doi.org/10.48550/arxiv.2608.10126","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2608.10126","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2608.10126","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2608.10126","pdf_url":"https://arxiv.org/pdf/2608.10126","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/10","score":0.4074000120162964,"display_name":"Reduced inequalities"}],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7202280720.pdf","grobid_xml":"https://content.openalex.org/works/W7202280720.grobid-xml"},"referenced_works_count":6,"referenced_works":["https://openalex.org/W2973379954","https://openalex.org/W4226278401","https://openalex.org/W4323929307","https://openalex.org/W4404783337","https://openalex.org/W4416097798","https://openalex.org/W4417248785"],"related_works":["https://openalex.org/W4412886789","https://openalex.org/W4415111447","https://openalex.org/W7151575003","https://openalex.org/W4391835636","https://openalex.org/W7106820257","https://openalex.org/W4396601201","https://openalex.org/W7139111931","https://openalex.org/W4414274825","https://openalex.org/W7131625403","https://openalex.org/W4399061767"],"abstract_inverted_index":{"Reinforcement":[0],"Learning":[1],"from":[2,90,105,122],"Human":[3],"Feedback":[4],"(RLHF)":[5],"aggregates":[6],"heterogeneous":[7],"preferences":[8,18,37],"into":[9],"a":[10,24,82],"single":[11],"reward":[12,33,54,78,127,146],"model,":[13],"assuming":[14],"preference":[15,30,51,64,74],"homogeneity.":[16],"When":[17],"are":[19,38],"heterogeneous,":[20],"this":[21,62],"aggregation":[22],"induces":[23],"procedural":[25,44,115],"fairness":[26,45,97,116],"failure":[27],"where":[28,144],"majority":[29],"groups":[31,104],"dominate":[32],"learning":[34,79],"while":[35],"minority":[36],"systematically":[39],"under-represented.":[40],"This":[41],"work":[42],"defines":[43],"in":[46,118,126,130],"alignment":[47,88,119],"as":[48],"preserving":[49],"distinct":[50],"signals":[52],"during":[53],"modeling":[55],"and":[56,94,101,141],"shows":[57],"that":[58,114],"standard":[59],"RLHF":[60,67],"violates":[61],"via":[63],"averaging.":[65],"Preference-Aware":[66],"(PA-RLHF)":[68],"is":[69],"introduced,":[70],"separating":[71],"optimization":[72],"across":[73,151],"modes":[75],"at":[76],"the":[77,96],"stage.":[80],"In":[81],"controlled":[83],"setting,":[84],"PA-RLHF":[85],"improves":[86],"overall":[87],"accuracy":[89],"46.9%":[91],"to":[92,107],"67.9%":[93],"reduces":[95],"gap":[98],"between":[99],"best":[100],"worst":[102],"aligned":[103],"15.9":[106],"9.6":[108],"percentage":[109],"points.":[110],"These":[111],"results":[112],"show":[113],"failures":[117],"can":[120,148],"arise":[121],"structural":[123],"design":[124],"choices":[125],"learning,":[128],"even":[129],"controlled,":[131],"noise-free":[132],"settings,":[133],"with":[134],"direct":[135],"implications":[136],"for":[137],"large":[138],"language":[139],"models":[140,147],"agentic":[142],"systems,":[143],"biased":[145],"compound":[149],"inequities":[150],"sequential":[152],"decisions.":[153]},"counts_by_year":[],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2026-08-13T00:00:00"}