{"id":"https://openalex.org/W4404782763","doi":"https://doi.org/10.18653/v1/2024.emnlp-main.252","title":"ARES: Alternating Reinforcement Learning and Supervised Fine-Tuning for Enhanced Multi-Modal Chain-of-Thought Reasoning Through Diverse AI Feedback","display_name":"ARES: Alternating Reinforcement Learning and Supervised Fine-Tuning for Enhanced Multi-Modal Chain-of-Thought Reasoning Through Diverse AI Feedback","publication_year":2024,"publication_date":"2024-01-01","ids":{"openalex":"https://openalex.org/W4404782763","doi":"https://doi.org/10.18653/v1/2024.emnlp-main.252"},"language":"en","primary_location":{"id":"doi:10.18653/v1/2024.emnlp-main.252","is_oa":true,"landing_page_url":"http://dx.doi.org/10.18653/v1/2024.emnlp-main.252","pdf_url":"https://aclanthology.org/2024.emnlp-main.252.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2024 Conference on Empirical Methods in Natural Language Processing","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://aclanthology.org/2024.emnlp-main.252.pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5048570379","display_name":"Ju-Seung Byun","orcid":null},"institutions":[{"id":"https://openalex.org/I52357470","display_name":"The Ohio State University","ror":"https://ror.org/00rs6vg23","country_code":"US","type":"education","lineage":["https://openalex.org/I52357470"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ju-Seung Byun","raw_affiliation_strings":["Department of Computer Science and Engineering The Ohio State University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering The Ohio State University","institution_ids":["https://openalex.org/I52357470"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5019424838","display_name":"Jiyun Chun","orcid":null},"institutions":[{"id":"https://openalex.org/I52357470","display_name":"The Ohio State University","ror":"https://ror.org/00rs6vg23","country_code":"US","type":"education","lineage":["https://openalex.org/I52357470"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jiyun Chun","raw_affiliation_strings":["Department of Computer Science and Engineering The Ohio State University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering The Ohio State University","institution_ids":["https://openalex.org/I52357470"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5089572620","display_name":"Jihyung Kil","orcid":null},"institutions":[{"id":"https://openalex.org/I52357470","display_name":"The Ohio State University","ror":"https://ror.org/00rs6vg23","country_code":"US","type":"education","lineage":["https://openalex.org/I52357470"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jihyung Kil","raw_affiliation_strings":["Department of Computer Science and Engineering The Ohio State University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering The Ohio State University","institution_ids":["https://openalex.org/I52357470"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5057049889","display_name":"Andrew Perrault","orcid":"https://orcid.org/0000-0002-5062-7958"},"institutions":[{"id":"https://openalex.org/I52357470","display_name":"The Ohio State University","ror":"https://ror.org/00rs6vg23","country_code":"US","type":"education","lineage":["https://openalex.org/I52357470"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Andrew Perrault","raw_affiliation_strings":["Department of Computer Science and Engineering The Ohio State University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering The Ohio State University","institution_ids":["https://openalex.org/I52357470"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I52357470"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.23296028,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"4410","last_page":"4430"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12805","display_name":"Cognitive Science and Mapping","score":0.7623999714851379,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12805","display_name":"Cognitive Science and Mapping","score":0.7623999714851379,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10320","display_name":"Neural Networks and Applications","score":0.7346000075340271,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.7484737038612366},{"id":"https://openalex.org/keywords/modal","display_name":"Modal","score":0.7173931002616882},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6134548187255859},{"id":"https://openalex.org/keywords/reinforcement","display_name":"Reinforcement","score":0.5319868326187134},{"id":"https://openalex.org/keywords/chain","display_name":"Chain (unit)","score":0.5004005432128906},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4600382447242737},{"id":"https://openalex.org/keywords/cognitive-science","display_name":"Cognitive science","score":0.32609307765960693},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.1740998923778534},{"id":"https://openalex.org/keywords/physics","display_name":"Physics","score":0.16686859726905823},{"id":"https://openalex.org/keywords/chemistry","display_name":"Chemistry","score":0.1507212519645691},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.14329737424850464}],"concepts":[{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.7484737038612366},{"id":"https://openalex.org/C71139939","wikidata":"https://www.wikidata.org/wiki/Q910194","display_name":"Modal","level":2,"score":0.7173931002616882},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6134548187255859},{"id":"https://openalex.org/C67203356","wikidata":"https://www.wikidata.org/wiki/Q1321905","display_name":"Reinforcement","level":2,"score":0.5319868326187134},{"id":"https://openalex.org/C199185054","wikidata":"https://www.wikidata.org/wiki/Q552299","display_name":"Chain (unit)","level":2,"score":0.5004005432128906},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4600382447242737},{"id":"https://openalex.org/C188147891","wikidata":"https://www.wikidata.org/wiki/Q147638","display_name":"Cognitive science","level":1,"score":0.32609307765960693},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.1740998923778534},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.16686859726905823},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.1507212519645691},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.14329737424850464},{"id":"https://openalex.org/C188027245","wikidata":"https://www.wikidata.org/wiki/Q750446","display_name":"Polymer chemistry","level":1,"score":0.0},{"id":"https://openalex.org/C66938386","wikidata":"https://www.wikidata.org/wiki/Q633538","display_name":"Structural engineering","level":1,"score":0.0},{"id":"https://openalex.org/C1276947","wikidata":"https://www.wikidata.org/wiki/Q333","display_name":"Astronomy","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.18653/v1/2024.emnlp-main.252","is_oa":true,"landing_page_url":"http://dx.doi.org/10.18653/v1/2024.emnlp-main.252","pdf_url":"https://aclanthology.org/2024.emnlp-main.252.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2024 Conference on Empirical Methods in Natural Language Processing","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.18653/v1/2024.emnlp-main.252","is_oa":true,"landing_page_url":"http://dx.doi.org/10.18653/v1/2024.emnlp-main.252","pdf_url":"https://aclanthology.org/2024.emnlp-main.252.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2024 Conference on Empirical Methods in Natural Language Processing","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4404782763.pdf","grobid_xml":"https://content.openalex.org/works/W4404782763.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4310083477","https://openalex.org/W2328553770","https://openalex.org/W2920061524","https://openalex.org/W1977959518","https://openalex.org/W2038908348","https://openalex.org/W2107890255","https://openalex.org/W2106552856","https://openalex.org/W2145821588","https://openalex.org/W2086122291","https://openalex.org/W1987513656"],"abstract_inverted_index":{"Large":[0],"Multimodal":[1],"Models":[2],"(LMMs)":[3],"excel":[4],"at":[5],"comprehending":[6],"human":[7],"instructions":[8],"and":[9,24,52,81,139,147,167],"demonstrate":[10,170],"remarkable":[11],"results":[12],"across":[13],"a":[14,72,101,199],"broad":[15],"spectrum":[16],"of":[17,61,173],"tasks.Reinforcement":[18],"Learning":[19,79],"from":[20],"Human":[21],"Feedback":[22,26],"(RLHF)":[23],"AI":[25,46],"(RLAIF)":[27],"further":[28],"refine":[29],"LLMs":[30],"by":[31,188],"aligning":[32],"them":[33],"with":[34],"specific":[35],"preferences.These":[36],"methods":[37],"primarily":[38],"use":[39],"ranking-based":[40],"feedback":[41,63,105],"for":[42,67,117,208],"entire":[43],"generations.With":[44],"advanced":[45],"models":[47,186],"(Teacher),":[48],"such":[49,143],"as":[50,144],"GPT-4":[51],"Claude":[53],"3":[54],"Opus,":[55],"we":[56,85,121,152,190],"can":[57],"request":[58],"various":[59],"types":[60],"detailed":[62],"that":[64,76,192],"are":[65],"expensive":[66],"humans":[68],"to":[69,89,96,108,125,169,184,198],"provide.We":[70],"propose":[71],"two-stage":[73],"algorithm":[74],"ARES":[75,176],"Alternates":[77],"REinforcement":[78],"(RL)":[80],"Supervised":[82],"Fine-Tuning":[83],"(SFT).First,":[84],"ask":[86,122],"the":[87,98,118,123,130,154,163,171,193,209],"Teacher":[88,124],"score":[90],"how":[91],"much":[92],"each":[93],"sentence":[94],"contributes":[95],"solving":[97],"problem":[99],"in":[100,202],"Chainof-Thought":[102],"(CoT).This":[103],"sentence-level":[104],"allows":[106],"us":[107],"consider":[109],"individual":[110],"valuable":[111],"segments,":[112],"providing":[113],"more":[114],"granular":[115],"rewards":[116],"RL":[119,131,133,155],"procedure.Second,":[120],"correct":[126],"wrong":[127],"reasoning":[128,196],"after":[129],"stage.The":[132],"procedure":[134],"requires":[135],"substantial":[136],"hyperparameter":[137],"tuning":[138],"often":[140],"generates":[141],"errors":[142],"repetitive":[145],"words":[146],"incomplete":[148],"sentences.With":[149],"correction":[150],"feedback,":[151],"stabilize":[153],"fine-tuned":[156],"model":[157],"through":[158],"SFT.We":[159],"conduct":[160],"experiments":[161],"on":[162,206],"multi-modal":[164,210],"datasets":[165],"ScienceQA":[166],"A-OKVQA":[168],"effectiveness":[172],"our":[174],"proposal.The":[175],"rationale":[177,195],"achieves":[178],"around":[179],"70%":[180],"win":[181],"rate":[182],"compared":[183],"baseline":[185],"judged":[187],"GPT-4o.Additionally,":[189],"observe":[191],"improved":[194],"leads":[197],"2.5%":[200],"increase":[201],"inference":[203],"answer":[204],"accuracy":[205],"average":[207],"datasets.":[211]},"counts_by_year":[],"updated_date":"2026-07-29T14:22:42.915294","created_date":"2025-10-10T00:00:00"}