{"id":"https://openalex.org/W6910697293","doi":"https://doi.org/10.48550/arxiv.2503.18712","title":"LLaVAction: evaluating and training multi-modal large language models for action understanding","display_name":"LLaVAction: evaluating and training multi-modal large language models for action understanding","publication_year":2025,"publication_date":"2025-03-24","ids":{"openalex":"https://openalex.org/W6910697293","doi":"https://doi.org/10.48550/arxiv.2503.18712"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2503.18712","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2503.18712","pdf_url":"https://arxiv.org/pdf/2503.18712","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2503.18712","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5032195423","display_name":"Haozhe Qi","orcid":"https://orcid.org/0009-0007-7399-923X","observed_orcids":["https://orcid.org/0009-0007-7399-923X"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Qi, Haozhe","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Ye, Shaokai","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ye, Shaokai","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5013861606","display_name":"Alexander Mathis","orcid":"https://orcid.org/0000-0002-3777-2202","observed_orcids":["https://orcid.org/0000-0002-3777-2202"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mathis, Alexander","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5025196309","display_name":"Mackenzie Weygandt Mathis","orcid":"https://orcid.org/0000-0001-7368-4456","observed_orcids":["https://orcid.org/0000-0001-7368-4456"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mathis, Mackenzie W.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":true,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9448000192642212,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9448000192642212,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10812","display_name":"Human Pose and Action Recognition","score":0.05270000174641609,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11398","display_name":"Hand Gesture Recognition Systems","score":0.0006000000284984708,"subfield":{"id":"https://openalex.org/subfields/1709","display_name":"Human-Computer Interaction"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/multimodal-large-language-models","display_name":"multimodal large language models","score":0.9620000123977661},{"id":"https://openalex.org/keywords/action-understanding","display_name":"action understanding","score":0.9430000185966492},{"id":"https://openalex.org/keywords/egocentric-action-recognition","display_name":"egocentric action recognition","score":0.847000002861023},{"id":"https://openalex.org/keywords/epic-kitchens-100","display_name":"EPIC-KITCHENS-100","score":0.972000002861023},{"id":"https://openalex.org/keywords/temporal-action-localization","display_name":"temporal action localization","score":0.6489999890327454},{"id":"https://openalex.org/keywords/human-behavior-understanding","display_name":"human behavior understanding","score":0.39399999380111694},{"id":"https://openalex.org/keywords/question-answering","display_name":"question answering","score":0.3790000081062317}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7699999809265137},{"id":"https://openalex.org/C2780791683","wikidata":"https://www.wikidata.org/wiki/Q846785","display_name":"Action (physics)","level":2,"score":0.7390999794006348},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6435999870300293},{"id":"https://openalex.org/C43521106","wikidata":"https://www.wikidata.org/wiki/Q2165493","display_name":"Pipeline (software)","level":2,"score":0.6367999911308289},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.5782999992370605},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.46369999647140503},{"id":"https://openalex.org/C2777735758","wikidata":"https://www.wikidata.org/wiki/Q817765","display_name":"Path (computing)","level":2,"score":0.4523000121116638},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.38269999623298645},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.3652999997138977},{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.3637000024318695},{"id":"https://openalex.org/C184337299","wikidata":"https://www.wikidata.org/wiki/Q1437428","display_name":"Semantics (computer science)","level":2,"score":0.36329999566078186},{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.35839998722076416},{"id":"https://openalex.org/C198531522","wikidata":"https://www.wikidata.org/wiki/Q485146","display_name":"Sample (material)","level":2,"score":0.31679999828338623},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.29809999465942383},{"id":"https://openalex.org/C2983448237","wikidata":"https://www.wikidata.org/wiki/Q1078276","display_name":"Language understanding","level":2,"score":0.2824000120162964},{"id":"https://openalex.org/C2987834672","wikidata":"https://www.wikidata.org/wiki/Q4677630","display_name":"Action recognition","level":3,"score":0.27950000762939453},{"id":"https://openalex.org/C22367795","wikidata":"https://www.wikidata.org/wiki/Q7625208","display_name":"Structured prediction","level":2,"score":0.2721000015735626},{"id":"https://openalex.org/C136197465","wikidata":"https://www.wikidata.org/wiki/Q1729295","display_name":"Variety (cybernetics)","level":2,"score":0.26170000433921814},{"id":"https://openalex.org/C177148314","wikidata":"https://www.wikidata.org/wiki/Q170084","display_name":"Generalization","level":2,"score":0.25609999895095825}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2503.18712","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2503.18712","pdf_url":"https://arxiv.org/pdf/2503.18712","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2503.18712","doi":"https://doi.org/10.48550/arxiv.2503.18712","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2503.18712","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2503.18712","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2503.18712","pdf_url":"https://arxiv.org/pdf/2503.18712","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"sustainable_development_goals_aurora":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Understanding":[0],"human":[1],"behavior":[2,11],"requires":[3],"measuring":[4],"behavioral":[5],"actions.":[6,86,146],"Due":[7],"to":[8,82,113,132,143,182],"its":[9],"complexity,":[10],"is":[12],"best":[13],"mapped":[14],"onto":[15],"a":[16,61,97,122,140,184],"rich,":[17],"semantic":[18],"structure":[19],"such":[20],"as":[21,77],"language.":[22],"Emerging":[23],"multimodal":[24],"large":[25],"language":[26],"models":[27,76,197],"(MLLMs)":[28],"are":[29,198],"promising":[30,185],"candidates,":[31],"but":[32],"their":[33],"fine-grained":[34],"action":[35,57,104,117,130,154,173,190],"understanding":[36,118],"ability":[37,152],"has":[38],"not":[39],"been":[40],"fully":[41],"examined.":[42],"In":[43],"this":[44],"work,":[45],"we":[46,69,89],"reformulate":[47],"EPIC-KITCHENS-100,":[48],"one":[49],"of":[50,93,153],"the":[51,84,91,150,194],"largest":[52],"and":[53,109,139,171,196],"most":[54],"challenging":[55],"egocentric":[56],"recognition":[58,174],"datasets,":[59],"into":[60],"MLLM":[62,161],"benchmark":[63],"(EPIC-KITCHENS-100-MQA).":[64],"We":[65,95,120],"demonstrate":[66],"that":[67,101,127,177],"when":[68],"sample":[70],"difficult":[71],"answers":[72],"based":[73],"on":[74,136,159,169],"specialist":[75],"distractors,":[78],"leading":[79],"MLLMs":[80,181],"struggle":[81],"recognize":[83],"correct":[85],"How":[87],"can":[88],"increase":[90],"performance":[92],"MLLMs?":[94],"curated":[96],"supervised":[98],"finetuning":[99],"dataset":[100],"includes":[102],"`hard'":[103],"recognition,":[105],"temporal":[106],"detection,":[107],"captioning,":[108],"free-form":[110],"question":[111],"answering":[112],"improve":[114],"models'":[115,134],"diverse":[116],"capabilities.":[119],"introduce":[121],"new":[123],"model":[124],"called":[125],"LLaVAction":[126,147],"adds":[128],"an":[129],"token":[131],"boost":[133],"attention":[135],"visual":[137],"tokens":[138],"two-stage":[141],"pipeline":[142],"obtain":[144],"structured":[145],"greatly":[148],"improves":[149],"MLLMs'":[151],"understanding,":[155],"achieving":[156],"strong":[157],"improvements":[158],"both":[160],"benchmarks":[162],"(21":[163],"points":[164],"in":[165],"accuracy":[166],"over":[167],"GPT-4o":[168],"EPIC-KITCHENS-100-MQA)":[170],"established":[172],"benchmarks,":[175],"suggesting":[176],"our":[178],"methods":[179],"prepare":[180],"be":[183],"path":[186],"forward":[187],"for":[188],"complex":[189],"tasks.":[191],"Code,":[192],"data,":[193],"benchmark,":[195],"available":[199],"at":[200],"https://github.com/AdaptiveMotorControlLab/LLaVAction.":[201]},"counts_by_year":[],"updated_date":"2026-10-16T09:59:22.235173","created_date":"2025-10-10T00:00:00"}