{"id":"https://openalex.org/W4389520260","doi":"https://doi.org/10.18653/v1/2023.findings-emnlp.754","title":"SteerLM: Attribute Conditioned SFT as an (User-Steerable) Alternative to RLHF","display_name":"SteerLM: Attribute Conditioned SFT as an (User-Steerable) Alternative to RLHF","publication_year":2023,"publication_date":"2023-01-01","ids":{"openalex":"https://openalex.org/W4389520260","doi":"https://doi.org/10.18653/v1/2023.findings-emnlp.754"},"language":"en","primary_location":{"id":"doi:10.18653/v1/2023.findings-emnlp.754","doi":"https://doi.org/10.18653/v1/2023.findings-emnlp.754","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2023.findings-emnlp.754","pdf_url":"https://aclanthology.org/2023.findings-emnlp.754.pdf","source":{"id":"https://openalex.org/S7407096085","display_name":"Findings of the Association for Computational Linguistics: EMNLP (Findings of EMNLP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: EMNLP 2023","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","is_traditional":true,"indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://aclanthology.org/2023.findings-emnlp.754.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Yi Dong","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yi Dong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101475389","display_name":"Zhilin Wang","orcid":"https://orcid.org/0000-0002-0188-0332","observed_orcids":["https://orcid.org/0000-0002-0188-0332","https://orcid.org/0009-0001-7101-1220","https://orcid.org/0009-0008-0743-5142"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhilin Wang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5111090196","display_name":"Makesh Narsimhan Sreedhar","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Makesh Sreedhar","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100548260","display_name":"Xianchao Wu","orcid":"https://orcid.org/0000-0002-6561-4789","observed_orcids":["https://orcid.org/0000-0002-6561-4789"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xianchao Wu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5048403564","display_name":"Oleksii Kuchaiev","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Oleksii Kuchaiev","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":9.7215,"has_fulltext":true,"cited_by_count":88,"citation_normalized_percentile":{"value":0.98737206,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":99,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"11275","last_page":"11288"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.5023999810218811,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.5023999810218811,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.23499999940395355,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.07000000029802322,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[],"concepts":[{"id":"https://openalex.org/C2781265381","wikidata":"https://www.wikidata.org/wiki/Q5710255","display_name":"Helpfulness","level":2,"score":0.8760867118835449},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8288062810897827},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.6592429876327515},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.6132105588912964},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.5565104484558105},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.5340396761894226},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5253089070320129},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.48133692145347595},{"id":"https://openalex.org/C2775924081","wikidata":"https://www.wikidata.org/wiki/Q55608371","display_name":"Control (management)","level":2,"score":0.4694601595401764},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.39127010107040405},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.18653/v1/2023.findings-emnlp.754","doi":"https://doi.org/10.18653/v1/2023.findings-emnlp.754","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2023.findings-emnlp.754","pdf_url":"https://aclanthology.org/2023.findings-emnlp.754.pdf","source":{"id":"https://openalex.org/S7407096085","display_name":"Findings of the Association for Computational Linguistics: EMNLP (Findings of EMNLP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: EMNLP 2023","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2310.05344","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2310.05344","pdf_url":"https://arxiv.org/pdf/2310.05344","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2310.05344","doi":"https://doi.org/10.48550/arxiv.2310.05344","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2310.05344","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.18653/v1/2023.findings-emnlp.754","doi":"https://doi.org/10.18653/v1/2023.findings-emnlp.754","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2023.findings-emnlp.754","pdf_url":"https://aclanthology.org/2023.findings-emnlp.754.pdf","source":{"id":"https://openalex.org/S7407096085","display_name":"Findings of the Association for Computational Linguistics: EMNLP (Findings of EMNLP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: EMNLP 2023","raw_type":"proceedings-article"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320309480","display_name":"Nvidia","ror":"https://ror.org/03jdj4y14"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4389520260.pdf","grobid_xml":"https://content.openalex.org/works/W4389520260.grobid-xml"},"referenced_works_count":34,"referenced_works":["https://openalex.org/W2626804490","https://openalex.org/W2736601468","https://openalex.org/W2902201791","https://openalex.org/W2950242852","https://openalex.org/W2963825865","https://openalex.org/W2974231335","https://openalex.org/W3001279689","https://openalex.org/W3030163527","https://openalex.org/W3082115681","https://openalex.org/W3089263616","https://openalex.org/W3198002980","https://openalex.org/W3203065320","https://openalex.org/W3205068155","https://openalex.org/W3212637808","https://openalex.org/W4223908421","https://openalex.org/W4226227340","https://openalex.org/W4226278401","https://openalex.org/W4283445891","https://openalex.org/W4286588337","https://openalex.org/W4296413526","https://openalex.org/W4309663019","https://openalex.org/W4361229539","https://openalex.org/W4362707064","https://openalex.org/W4365211596","https://openalex.org/W4366196653","https://openalex.org/W4366341216","https://openalex.org/W4380993527","https://openalex.org/W4384918448","https://openalex.org/W4385572634","https://openalex.org/W4385734232","https://openalex.org/W7133189643","https://openalex.org/W7133208539","https://openalex.org/W7133220558","https://openalex.org/W7133233472"],"related_works":["https://openalex.org/W6996730461","https://openalex.org/W7155086127","https://openalex.org/W4415799187","https://openalex.org/W4417112382","https://openalex.org/W4412945014","https://openalex.org/W4415800425","https://openalex.org/W4402670459","https://openalex.org/W4415343757","https://openalex.org/W4387560550","https://openalex.org/W7010917313"],"abstract_inverted_index":{"Model":[0],"alignment":[1],"with":[2,18,52,155],"human":[3,19,31,146],"preferences":[4],"is":[5],"an":[6,110],"essential":[7],"step":[8],"in":[9,64],"making":[10],"Large":[11],"Language":[12],"Models":[13],"(LLMs)":[14],"helpful":[15,125],"and":[16,27,45,85,126,147],"consistent":[17],"values.It":[20],"typically":[21],"consists":[22],"of":[23,115,123],"supervised":[24,94],"fine-tuning":[25],"(SFT)":[26],"reinforcement":[28],"learning":[29],"from":[30,40],"feedback":[32,71],"(RLHF)":[33],"stages.However,":[34],"RLHF":[35,65,156],"faces":[36],"inherent":[37],"limitations":[38],"stemming":[39],"a":[41,93,119],"complex":[42],"training":[43],"setup":[44],"its":[46],"tendency":[47],"to":[48,74,100,107,109,150,161],"align":[49],"the":[50],"model":[51],"implicit":[53],"values":[54],"that":[55,78,97,133,142],"end":[56],"users":[57],"cannot":[58],"control":[59,101],"at":[60,164],"run-time.Moreover,":[61],"reward":[62],"models":[63],"stage":[66],"commonly":[67],"rely":[68],"on":[69,136],"single-dimensional":[70],"as":[72,82],"opposed":[73],"explicit,":[75],"multifaceted":[76],"signals":[77],"indicate":[79],"attributes":[80],"such":[81],"helpfulness,":[83],"humor,":[84],"toxicity.To":[86],"address":[87],"these":[88],"limitations,":[89],"we":[90],"propose":[91],"STEERLM,":[92],"finetuning":[95],"method":[96],"empowers":[98],"end-users":[99],"responses":[102,106,128,141],"during":[103],"inference.STEERLM":[104],"conditions":[105],"conform":[108],"explicitly":[111],"defined":[112],"multi-dimensional":[113],"set":[114],"attributes,":[116],"thereby":[117],"empowering":[118],"steerable":[120],"AI":[121],"capable":[122],"generating":[124],"high-quality":[127],"while":[129,157],"maintaining":[130],"customizability.Experiments":[131],"show":[132],"STEERLM":[134,163],"trained":[135,154],"open":[137],"source":[138],"datasets":[139],"generates":[140],"are":[143],"preferred":[144],"by":[145],"automatic":[148],"evaluators":[149],"many":[151],"state-of-the-art":[152],"baselines":[153],"being":[158],"much":[159],"easier":[160],"train.Try":[162],"https://huggingface.co/":[165],"nvidia/SteerLM-llama2-13B":[166]},"counts_by_year":[{"year":2026,"cited_by_count":23},{"year":2025,"cited_by_count":31},{"year":2024,"cited_by_count":34}],"updated_date":"2026-11-10T11:39:43.021295","created_date":"2023-12-11T00:00:00"}