{"id":"https://openalex.org/W4406314914","doi":"https://doi.org/10.48550/arxiv.2501.06186","title":"LlamaV-o1: Rethinking Step-by-step Visual Reasoning in LLMs","display_name":"LlamaV-o1: Rethinking Step-by-step Visual Reasoning in LLMs","publication_year":2025,"publication_date":"2025-01-10","ids":{"openalex":"https://openalex.org/W4406314914","doi":"https://doi.org/10.48550/arxiv.2501.06186"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2501.06186","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2501.06186","pdf_url":"https://arxiv.org/pdf/2501.06186","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2501.06186","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5074650489","display_name":"Omkar Thawakar","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210113480","display_name":"Mohamed bin Zayed University of Artificial Intelligence","ror":"https://ror.org/0258gkt32","country_code":"AE","type":"education","lineage":["https://openalex.org/I4210113480"]}],"countries":["AE"],"is_corresponding":false,"raw_author_name":"Thawakar, Omkar","raw_affiliation_strings":["Mohamed bin Zayed University of AI ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Mohamed bin Zayed University of AI ,","institution_ids":["https://openalex.org/I4210113480"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5108371642","display_name":"Dinura Dissanayake","orcid":"https://orcid.org/0009-0007-6339-1538","observed_orcids":["https://orcid.org/0009-0007-6339-1538"]},"institutions":[{"id":"https://openalex.org/I4210113480","display_name":"Mohamed bin Zayed University of Artificial Intelligence","ror":"https://ror.org/0258gkt32","country_code":"AE","type":"education","lineage":["https://openalex.org/I4210113480"]}],"countries":["AE"],"is_corresponding":false,"raw_author_name":"Dissanayake, Dinura","raw_affiliation_strings":["Mohamed bin Zayed University of AI ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Mohamed bin Zayed University of AI ,","institution_ids":["https://openalex.org/I4210113480"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5115002295","display_name":"Ketan More","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210113480","display_name":"Mohamed bin Zayed University of Artificial Intelligence","ror":"https://ror.org/0258gkt32","country_code":"AE","type":"education","lineage":["https://openalex.org/I4210113480"]}],"countries":["AE"],"is_corresponding":false,"raw_author_name":"More, Ketan","raw_affiliation_strings":["Mohamed bin Zayed University of AI ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Mohamed bin Zayed University of AI ,","institution_ids":["https://openalex.org/I4210113480"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5115860474","display_name":"Ritesh Thawkar","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210113480","display_name":"Mohamed bin Zayed University of Artificial Intelligence","ror":"https://ror.org/0258gkt32","country_code":"AE","type":"education","lineage":["https://openalex.org/I4210113480"]}],"countries":["AE"],"is_corresponding":false,"raw_author_name":"Thawkar, Ritesh","raw_affiliation_strings":["Mohamed bin Zayed University of AI ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Mohamed bin Zayed University of AI ,","institution_ids":["https://openalex.org/I4210113480"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5093925844","display_name":"Ahmed Heakl","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210113480","display_name":"Mohamed bin Zayed University of Artificial Intelligence","ror":"https://ror.org/0258gkt32","country_code":"AE","type":"education","lineage":["https://openalex.org/I4210113480"]}],"countries":["AE"],"is_corresponding":false,"raw_author_name":"Heakl, Ahmed","raw_affiliation_strings":["Mohamed bin Zayed University of AI ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Mohamed bin Zayed University of AI ,","institution_ids":["https://openalex.org/I4210113480"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5114415249","display_name":"Noor Ahsan","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210113480","display_name":"Mohamed bin Zayed University of Artificial Intelligence","ror":"https://ror.org/0258gkt32","country_code":"AE","type":"education","lineage":["https://openalex.org/I4210113480"]}],"countries":["AE"],"is_corresponding":false,"raw_author_name":"Ahsan, Noor","raw_affiliation_strings":["Mohamed bin Zayed University of AI ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Mohamed bin Zayed University of AI ,","institution_ids":["https://openalex.org/I4210113480"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Li, Yuhao","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210113480","display_name":"Mohamed bin Zayed University of Artificial Intelligence","ror":"https://ror.org/0258gkt32","country_code":"AE","type":"education","lineage":["https://openalex.org/I4210113480"]}],"countries":["AE"],"is_corresponding":false,"raw_author_name":"Li, Yuhao","raw_affiliation_strings":["Mohamed bin Zayed University of AI ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Mohamed bin Zayed University of AI ,","institution_ids":["https://openalex.org/I4210113480"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5116027349","display_name":"Mohammed Zumri","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210113480","display_name":"Mohamed bin Zayed University of Artificial Intelligence","ror":"https://ror.org/0258gkt32","country_code":"AE","type":"education","lineage":["https://openalex.org/I4210113480"]}],"countries":["AE"],"is_corresponding":false,"raw_author_name":"Zumri, Mohammed","raw_affiliation_strings":["Mohamed bin Zayed University of AI ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Mohamed bin Zayed University of AI ,","institution_ids":["https://openalex.org/I4210113480"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5049890269","display_name":"Jean Lahoud","orcid":"https://orcid.org/0000-0003-0315-6484","observed_orcids":["https://orcid.org/0000-0003-0315-6484"]},"institutions":[{"id":"https://openalex.org/I4210113480","display_name":"Mohamed bin Zayed University of Artificial Intelligence","ror":"https://ror.org/0258gkt32","country_code":"AE","type":"education","lineage":["https://openalex.org/I4210113480"]}],"countries":["AE"],"is_corresponding":false,"raw_author_name":"Lahoud, Jean","raw_affiliation_strings":["Mohamed bin Zayed University of AI ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Mohamed bin Zayed University of AI ,","institution_ids":["https://openalex.org/I4210113480"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5022929869","display_name":"Rao Muhammad Anwer","orcid":"https://orcid.org/0000-0002-9041-2214","observed_orcids":["https://orcid.org/0000-0002-9041-2214"]},"institutions":[{"id":"https://openalex.org/I4210113480","display_name":"Mohamed bin Zayed University of Artificial Intelligence","ror":"https://ror.org/0258gkt32","country_code":"AE","type":"education","lineage":["https://openalex.org/I4210113480"]}],"countries":["AE"],"is_corresponding":false,"raw_author_name":"Anwer, Rao Muhammad","raw_affiliation_strings":["Mohamed bin Zayed University of AI ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Mohamed bin Zayed University of AI ,","institution_ids":["https://openalex.org/I4210113480"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5009362997","display_name":"Hisham Cholakkal","orcid":"https://orcid.org/0000-0002-8230-9065","observed_orcids":["https://orcid.org/0000-0002-8230-9065"]},"institutions":[{"id":"https://openalex.org/I4210113480","display_name":"Mohamed bin Zayed University of Artificial Intelligence","ror":"https://ror.org/0258gkt32","country_code":"AE","type":"education","lineage":["https://openalex.org/I4210113480"]}],"countries":["AE"],"is_corresponding":false,"raw_author_name":"Cholakkal, Hisham","raw_affiliation_strings":["Mohamed bin Zayed University of AI ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Mohamed bin Zayed University of AI ,","institution_ids":["https://openalex.org/I4210113480"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5087781064","display_name":"Ivan Laptev","orcid":"https://orcid.org/0000-0001-7072-3325","observed_orcids":["https://orcid.org/0000-0001-7072-3325"]},"institutions":[{"id":"https://openalex.org/I4210113480","display_name":"Mohamed bin Zayed University of Artificial Intelligence","ror":"https://ror.org/0258gkt32","country_code":"AE","type":"education","lineage":["https://openalex.org/I4210113480"]}],"countries":["AE"],"is_corresponding":false,"raw_author_name":"Laptev, Ivan","raw_affiliation_strings":["Mohamed bin Zayed University of AI ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Mohamed bin Zayed University of AI ,","institution_ids":["https://openalex.org/I4210113480"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103215307","display_name":"Mubarak Shah","orcid":"https://orcid.org/0000-0002-8216-1128","observed_orcids":["https://orcid.org/0000-0002-8216-1128","https://orcid.org/0000-0001-6172-5572"]},"institutions":[{"id":"https://openalex.org/I106165777","display_name":"University of Central Florida","ror":"https://ror.org/036nfer12","country_code":"US","type":"education","lineage":["https://openalex.org/I106165777"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Shah, Mubarak","raw_affiliation_strings":["University of Central Florida ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Central Florida ,","institution_ids":["https://openalex.org/I106165777"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100760570","display_name":"Fahad Shahbaz Khan","orcid":"https://orcid.org/0000-0002-4263-3143","observed_orcids":["https://orcid.org/0000-0002-4263-3143"]},"institutions":[{"id":"https://openalex.org/I102134673","display_name":"Link\u00f6ping University","ror":"https://ror.org/05ynxx418","country_code":"SE","type":"education","lineage":["https://openalex.org/I102134673"]},{"id":"https://openalex.org/I4210113480","display_name":"Mohamed bin Zayed University of Artificial Intelligence","ror":"https://ror.org/0258gkt32","country_code":"AE","type":"education","lineage":["https://openalex.org/I4210113480"]}],"countries":["AE","SE"],"is_corresponding":false,"raw_author_name":"Khan, Fahad Shahbaz","raw_affiliation_strings":["Link\u00f6ping University ,","Mohamed bin Zayed University of AI ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Link\u00f6ping University ,","institution_ids":["https://openalex.org/I102134673"]},{"raw_affiliation_string":"Mohamed bin Zayed University of AI ,","institution_ids":["https://openalex.org/I4210113480"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5090790109","display_name":"Salman Khan","orcid":"https://orcid.org/0000-0002-8070-6577","observed_orcids":["https://orcid.org/0000-0002-8070-6577"]},"institutions":[{"id":"https://openalex.org/I118347636","display_name":"Australian National University","ror":"https://ror.org/019wvm592","country_code":"AU","type":"education","lineage":["https://openalex.org/I118347636"]},{"id":"https://openalex.org/I4210113480","display_name":"Mohamed bin Zayed University of Artificial Intelligence","ror":"https://ror.org/0258gkt32","country_code":"AE","type":"education","lineage":["https://openalex.org/I4210113480"]}],"countries":["AE","AU"],"is_corresponding":false,"raw_author_name":"Khan, Salman","raw_affiliation_strings":["Australian National University LlamaV-o1 Project :","Mohamed bin Zayed University of AI ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Australian National University LlamaV-o1 Project :","institution_ids":["https://openalex.org/I118347636"]},{"raw_affiliation_string":"Mohamed bin Zayed University of AI ,","institution_ids":["https://openalex.org/I4210113480"]}]}],"institutions":[],"countries_distinct_count":4,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.0,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.7210000157356262},{"id":"https://openalex.org/keywords/curriculum-learning","display_name":"curriculum learning","score":0.9340000152587891},{"id":"https://openalex.org/keywords/scientific-reasoning","display_name":"scientific reasoning","score":0.5799999833106995}],"concepts":[{"id":"https://openalex.org/C3019136120","wikidata":"https://www.wikidata.org/wiki/Q2462428","display_name":"Two step","level":2,"score":0.7101073265075684},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.39979884028434753},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.08010601997375488},{"id":"https://openalex.org/C21951064","wikidata":"https://www.wikidata.org/wiki/Q899212","display_name":"Combinatorial chemistry","level":1,"score":0.05871689319610596}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2501.06186","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2501.06186","pdf_url":"https://arxiv.org/pdf/2501.06186","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2501.06186","doi":"https://doi.org/10.48550/arxiv.2501.06186","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2501.06186","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2501.06186","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2501.06186","pdf_url":"https://arxiv.org/pdf/2501.06186","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4406314914.pdf","grobid_xml":"https://content.openalex.org/works/W4406314914.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4417286102","https://openalex.org/W7160207210","https://openalex.org/W6891885324","https://openalex.org/W7110994293","https://openalex.org/W7162606446","https://openalex.org/W4417128639","https://openalex.org/W4416621882","https://openalex.org/W4417087233","https://openalex.org/W4417447125","https://openalex.org/W7131683544"],"abstract_inverted_index":{"Reasoning":[0],"is":[1,18,186],"a":[2,23,41,61,75,118,156,166,195],"fundamental":[3],"capability":[4],"for":[5,26,44,188],"solving":[6],"complex":[7,86],"multi-step":[8,69,167,189],"problems,":[9],"particularly":[10],"in":[11,49,97],"visual":[12,28,47,62,87,110,123,159],"contexts":[13],"where":[14,171],"sequential":[15],"step-wise":[16,34],"understanding":[17],"essential.":[19],"Existing":[20],"approaches":[21],"lack":[22],"comprehensive":[24,42],"framework":[25,43],"evaluating":[27],"reasoning":[29,48,63,70,91,95,111,124,145,160,190],"and":[30,108,135,181,191,209,249],"do":[31],"not":[32],"emphasize":[33],"problem-solving.":[35,182],"To":[36],"this":[37],"end,":[38],"we":[39,59,116,154],"propose":[40,117],"advancing":[45],"step-by-step":[46,193],"large":[50],"language":[51],"models":[52,208],"(LMMs)":[53],"through":[54,194],"three":[55],"key":[56],"contributions.":[57],"First,":[58],"introduce":[60],"benchmark":[64,73],"specifically":[65],"designed":[66,187],"to":[67,89,105,148,176,217],"evaluate":[68],"tasks.":[71],"The":[72,138,183],"presents":[74],"diverse":[76],"set":[77],"of":[78,102,129,227,233],"challenges":[79],"with":[80,92,229],"eight":[81],"different":[82],"categories":[83],"ranging":[84],"from":[85],"perception":[88],"scientific":[90],"over":[93],"4k":[94],"steps":[96],"total,":[98],"enabling":[99],"robust":[100],"evaluation":[101],"LLMs'":[103],"abilities":[104],"perform":[106],"accurate":[107],"interpretable":[109],"across":[112,235],"multiple":[113],"steps.":[114],"Second,":[115],"novel":[119],"metric":[120,140],"that":[121,202],"assesses":[122],"quality":[125],"at":[126],"the":[127,218],"granularity":[128],"individual":[130],"steps,":[131],"emphasizing":[132],"both":[133],"correctness":[134],"logical":[136],"coherence.":[137],"proposed":[139,184],"offers":[141],"deeper":[142],"insights":[143],"into":[144],"performance":[146],"compared":[147],"traditional":[149],"end-task":[150],"accuracy":[151],"metrics.":[152],"Third,":[153],"present":[155],"new":[157],"multimodal":[158],"model,":[161,248],"named":[162],"LlamaV-o1,":[163],"trained":[164],"using":[165],"curriculum":[168],"learning":[169],"approach,":[170],"tasks":[172],"are":[173,251],"progressively":[174],"organized":[175],"facilitate":[177],"incremental":[178],"skill":[179],"acquisition":[180],"LlamaV-o1":[185,204,222],"learns":[192],"structured":[196],"training":[197],"paradigm.":[198],"Extensive":[199],"experiments":[200],"show":[201],"our":[203,221],"outperforms":[205],"existing":[206],"open-source":[207],"performs":[210],"favorably":[211],"against":[212],"close-source":[213],"proprietary":[214],"models.":[215],"Compared":[216],"recent":[219],"Llava-CoT,":[220],"achieves":[223],"an":[224,230],"average":[225],"score":[226],"67.3":[228],"absolute":[231],"gain":[232],"3.8\\%":[234],"six":[236],"benchmarks":[237],"while":[238],"being":[239],"5":[240],"times":[241],"faster":[242],"during":[243],"inference":[244],"scaling.":[245],"Our":[246],"benchmark,":[247],"code":[250],"publicly":[252],"available.":[253]},"counts_by_year":[],"updated_date":"2026-11-01T11:40:01.658920","created_date":"2025-01-14T00:00:00"}