{"id":"https://openalex.org/W4412889876","doi":"https://doi.org/10.18653/v1/2025.acl-long.1308","title":"From Lists to Emojis: How Format Bias Affects Model Alignment","display_name":"From Lists to Emojis: How Format Bias Affects Model Alignment","publication_year":2025,"publication_date":"2025-01-01","ids":{"openalex":"https://openalex.org/W4412889876","doi":"https://doi.org/10.18653/v1/2025.acl-long.1308"},"language":"en","primary_location":{"id":"doi:10.18653/v1/2025.acl-long.1308","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.acl-long.1308","pdf_url":"https://aclanthology.org/2025.acl-long.1308.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 63rd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://aclanthology.org/2025.acl-long.1308.pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5022236206","display_name":"Xuanchang Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xuanchang Zhang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5036346400","display_name":"Wei Xiong","orcid":"https://orcid.org/0000-0002-8826-2245"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wei Xiong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101668648","display_name":"Lichang Chen","orcid":"https://orcid.org/0000-0003-1173-7131"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lichang Chen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5039076312","display_name":"Tianyi Zhou","orcid":"https://orcid.org/0000-0001-5348-0632"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tianyi Zhou","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5081493266","display_name":"Heng Huang","orcid":"https://orcid.org/0000-0001-5552-3333"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Heng Huang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5114833240","display_name":"Tong Zhang","orcid":"https://orcid.org/0000-0001-8163-3050"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tong Zhang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"26940","last_page":"26961"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9898999929428101,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9898999929428101,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12380","display_name":"Authorship Attribution and Profiling","score":0.9814000129699707,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13629","display_name":"Text Readability and Simplification","score":0.9571999907493591,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6859835386276245},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.3650965094566345},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3607907295227051}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6859835386276245},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3650965094566345},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3607907295227051}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.18653/v1/2025.acl-long.1308","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.acl-long.1308","pdf_url":"https://aclanthology.org/2025.acl-long.1308.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 63rd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.18653/v1/2025.acl-long.1308","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.acl-long.1308","pdf_url":"https://aclanthology.org/2025.acl-long.1308.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 63rd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)","raw_type":"proceedings-article"},"sustainable_development_goals":[{"display_name":"Reduced inequalities","score":0.5099999904632568,"id":"https://metadata.un.org/sdg/10"}],"awards":[{"id":"https://openalex.org/G1616478382","display_name":null,"funder_award_id":"CCF 2348306","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G2721289185","display_name":"III: Medium: Collaborative Research: Integrating Large-Scale Machine Learning and Edge Computing for Collaborative Autonomous Vehicles","funder_award_id":"2348169","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G2869857317","display_name":null,"funder_award_id":"DBI 2405416","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G3782954227","display_name":"Collaborative Research: CCRI: New: A Scalable Hardware and Software Environment Enabling Secure Multi-party Learning","funder_award_id":"2347617","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G5995598075","display_name":null,"funder_award_id":"CNS 2347617","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G6301008065","display_name":"A New Machine Learning Framework for Single-Cell Multi-Omics Bioinformatics","funder_award_id":"2405416","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G7849508813","display_name":"Collaborative Research: PPoSS: LARGE: Co-designing  Hardware, Software, and Algorithms to Enable Extreme-Scale Machine Learning Systems","funder_award_id":"2348306","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G8057332120","display_name":null,"funder_award_id":"IIS 2347592","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G8667467031","display_name":"Collaborative Research: III: Medium: New Machine Learning Empowered Nanoinformatics System for Advancing Nanomaterial Design","funder_award_id":"2347592","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4412889876.pdf","grobid_xml":"https://content.openalex.org/works/W4412889876.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W4391913857","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W3204019825"],"abstract_inverted_index":{"In":[0],"this":[1,100],"paper,":[2],"we":[3,102,128,141],"study":[4,105],"format":[5,34,93,126,151,178,192],"biases":[6,31,51,94,107,152],"in":[7,108],"reinforcement":[8],"learning":[9,110],"from":[10],"human":[11,20],"feedback":[12],"(RLHF).We":[13],"observe":[14],"that":[15,77,130],"many":[16],"widely-used":[17],"preference":[18,72,109],"models-including":[19],"evaluators,":[21],"GPT-4,":[22],"and":[23,42,61,166,193,200],"top-ranking":[24],"models":[25,46,73],"on":[26,56],"the":[27,104,112,147,177,182,188],"RewardBench":[28],"benchmark-exhibit":[29],"strong":[30],"towards":[32],"specific":[33],"patterns,":[35],"such":[36,162],"as":[37,163,170],"lists,":[38],"links,":[39],"bold":[40],"text,":[41],"emojis.Furthermore,":[43],"large":[44],"language":[45],"(LLMs)":[47],"can":[48,142,153],"exploit":[49],"these":[50,150],"to":[52,87,175,180,190],"achieve":[53],"higher":[54],"rankings":[55],"popular":[57],"benchmarks":[58],"like":[59],"AlpacaEval":[60],"LMSYS":[62],"Chatbot":[63],"Arena.One":[64],"notable":[65],"example":[66],"is":[67,85,172],"verbosity":[68,96],"bias,":[69,116],"where":[70],"current":[71],"favor":[74],"longer":[75],"responses":[76],"appear":[78],"more":[79],"comprehensive,":[80],"even":[81],"when":[82],"their":[83],"quality":[84,183],"equal":[86],"or":[88],"lower":[89],"than":[90,139,179],"shorter":[91],"responses.However,":[92],"beyond":[95,111],"remain":[97],"largely":[98],"underexplored.In":[99],"work,":[101],"extend":[103],"of":[106,121,125,135,184],"commonly":[113],"recognized":[114],"length":[115],"offering":[117],"a":[118,122,132],"comprehensive":[119],"analysis":[120],"wider":[123],"range":[124],"biases.Additionally,":[127],"show":[129],"with":[131],"small":[133],"amount":[134],"biased":[136],"data":[137],"(less":[138],"1%),":[140],"inject":[143],"significant":[144],"bias":[145],"into":[146],"reward":[148],"model.Moreover,":[149],"also":[154],"be":[155],"easily":[156],"exploited":[157],"by":[158],"downstream":[159],"alignment":[160,198],"algorithms,":[161],"best-of-n":[164],"sampling":[165],"online":[167],"iterative":[168],"DPO,":[169],"it":[171],"usually":[173],"easier":[174],"manipulate":[176],"improve":[181],"responses.Our":[185],"findings":[186],"emphasize":[187],"need":[189],"disentangle":[191],"content":[194],"both":[195],"for":[196],"designing":[197],"algorithms":[199],"evaluating":[201],"models.":[202]},"counts_by_year":[],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}