{"id":"https://openalex.org/W4402670320","doi":"https://doi.org/10.18653/v1/2024.findings-acl.517","title":"TempCompass: Do Video LLMs Really Understand Videos?","display_name":"TempCompass: Do Video LLMs Really Understand Videos?","publication_year":2024,"publication_date":"2024-01-01","ids":{"openalex":"https://openalex.org/W4402670320","doi":"https://doi.org/10.18653/v1/2024.findings-acl.517"},"language":"en","primary_location":{"id":"doi:10.18653/v1/2024.findings-acl.517","doi":"https://doi.org/10.18653/v1/2024.findings-acl.517","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2024.findings-acl.517","pdf_url":"https://aclanthology.org/2024.findings-acl.517.pdf","source":{"id":"https://openalex.org/S4363605144","display_name":"Findings of the Association for Computational Linguistics: ACL (Findings of ACL)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics ACL 2024","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","is_traditional":true,"indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://aclanthology.org/2024.findings-acl.517.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5044541656","display_name":"Yuanxin Liu","orcid":"https://orcid.org/0000-0002-0673-4887","observed_orcids":["https://orcid.org/0000-0002-0673-4887"]},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yuanxin Liu","raw_affiliation_strings":["National Key Laboratory for Multimedia Information Processing , School of Computer Science , Peking University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National Key Laboratory for Multimedia Information Processing , School of Computer Science , Peking University","institution_ids":["https://openalex.org/I20231570"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100723941","display_name":"Shicheng Li","orcid":"https://orcid.org/0000-0002-5724-0641","observed_orcids":["https://orcid.org/0000-0002-5724-0641"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shicheng Li","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5085993597","display_name":"Yi Liu","orcid":"https://orcid.org/0000-0003-4980-7916","observed_orcids":["https://orcid.org/0000-0003-4980-7916"]},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yi Liu","raw_affiliation_strings":["National Key Laboratory for Multimedia Information Processing , School of Computer Science , Peking University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National Key Laboratory for Multimedia Information Processing , School of Computer Science , Peking University","institution_ids":["https://openalex.org/I20231570"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100394392","display_name":"Yu-Xiang Wang","orcid":"https://orcid.org/0000-0002-6403-212X","observed_orcids":["https://orcid.org/0000-0002-6403-212X"]},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yuxiang Wang","raw_affiliation_strings":["National Key Laboratory for Multimedia Information Processing , School of Computer Science , Peking University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National Key Laboratory for Multimedia Information Processing , School of Computer Science , Peking University","institution_ids":["https://openalex.org/I20231570"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101030723","display_name":"Shuhuai Ren","orcid":"https://orcid.org/0009-0001-9998-864X","observed_orcids":["https://orcid.org/0009-0001-9998-864X"]},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Shuhuai Ren","raw_affiliation_strings":["National Key Laboratory for Multimedia Information Processing , School of Computer Science , Peking University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National Key Laboratory for Multimedia Information Processing , School of Computer Science , Peking University","institution_ids":["https://openalex.org/I20231570"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100385654","display_name":"Lei Li","orcid":"https://orcid.org/0000-0001-7927-233X","observed_orcids":["https://orcid.org/0000-0001-7927-233X"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lei Li","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5113392191","display_name":"Sishuo Chen","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]},{"id":"https://openalex.org/I889458895","display_name":"University of Hong Kong","ror":"https://ror.org/02zhqgq86","country_code":"HK","type":"education","lineage":["https://openalex.org/I889458895"]}],"countries":["CN","HK"],"is_corresponding":false,"raw_author_name":"Sishuo Chen","raw_affiliation_strings":["Center for Data Science , Peking University \u2020 The University of Hong Kong"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Center for Data Science , Peking University \u2020 The University of Hong Kong","institution_ids":["https://openalex.org/I20231570","https://openalex.org/I889458895"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Xu Sun","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xu Sun","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5101663633","display_name":"Lu Hou","orcid":"https://orcid.org/0000-0002-4694-1821","observed_orcids":["https://orcid.org/0000-0002-4694-1821","https://orcid.org/0000-0003-3085-9353"]},"institutions":[{"id":"https://openalex.org/I2250955327","display_name":"Huawei Technologies (China)","ror":"https://ror.org/00cmhce21","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250955327"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Lu Hou","raw_affiliation_strings":["Huawei Noah's Ark Lab"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Huawei Noah's Ark Lab","institution_ids":["https://openalex.org/I2250955327"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":40.1086,"has_fulltext":true,"cited_by_count":285,"citation_normalized_percentile":{"value":0.99788094,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":99,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"8731","last_page":"8772"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10812","display_name":"Human Pose and Action Recognition","score":0.0,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10775","display_name":"Generative Adversarial Networks and Image Synthesis","score":0.0,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/video-large-language-models","display_name":"video large language models","score":0.9950000047683716},{"id":"https://openalex.org/keywords/time-perception","display_name":"time perception","score":0.9700000286102295}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5869397521018982},{"id":"https://openalex.org/C121684516","wikidata":"https://www.wikidata.org/wiki/Q7600677","display_name":"Computer graphics (images)","level":1,"score":0.34761518239974976},{"id":"https://openalex.org/C49774154","wikidata":"https://www.wikidata.org/wiki/Q131765","display_name":"Multimedia","level":1,"score":0.34033918380737305}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.18653/v1/2024.findings-acl.517","doi":"https://doi.org/10.18653/v1/2024.findings-acl.517","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2024.findings-acl.517","pdf_url":"https://aclanthology.org/2024.findings-acl.517.pdf","source":{"id":"https://openalex.org/S4363605144","display_name":"Findings of the Association for Computational Linguistics: ACL (Findings of ACL)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics ACL 2024","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2403.00476","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2403.00476","pdf_url":"https://arxiv.org/pdf/2403.00476","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2403.00476","doi":"https://doi.org/10.48550/arxiv.2403.00476","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2403.00476","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.18653/v1/2024.findings-acl.517","doi":"https://doi.org/10.18653/v1/2024.findings-acl.517","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2024.findings-acl.517","pdf_url":"https://aclanthology.org/2024.findings-acl.517.pdf","source":{"id":"https://openalex.org/S4363605144","display_name":"Findings of the Association for Computational Linguistics: ACL (Findings of ACL)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics ACL 2024","raw_type":"proceedings-article"},"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G4370721888","display_name":null,"funder_award_id":"62176002","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320335787","display_name":"Fundamental Research Funds for the Central Universities","ror":null}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4402670320.pdf","grobid_xml":"https://content.openalex.org/works/W4402670320.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4415023054","https://openalex.org/W7138456327","https://openalex.org/W7207016163","https://openalex.org/W7167663442","https://openalex.org/W4403344663","https://openalex.org/W7206171562","https://openalex.org/W7127448947","https://openalex.org/W4392011833","https://openalex.org/W7160872185","https://openalex.org/W4414970556"],"abstract_inverted_index":{"Recently,":[0],"there":[1],"is":[2],"a":[3,19,101,135,157,165],"surge":[4],"in":[5,64,134],"interest":[6],"surrounding":[7],"video":[8,120,166],"large":[9],"language":[10,148],"models":[11,210],"(Video":[12],"LLMs).However,":[13],"existing":[14],"benchmarks":[15],"fail":[16],"to":[17,38,179],"provide":[18],"comprehensive":[20],"feedback":[21],"on":[22,54,189],"the":[23,30,51,58,65,76,96,128,152,172,184,205],"temporal":[24,42,80,104,137,214],"perception":[25,81,215],"ability":[26],"of":[27,34,67,78,88,103],"Video":[28,141,187,197],"LLMs.On":[29],"one":[31],"hand,":[32,60],"most":[33],"them":[35],"are":[36,62,221],"unable":[37],"distinguish":[39],"between":[40],"different":[41,86],"aspects":[43,105],"(e.g.,":[44,70],"speed,":[45],"direction)":[46],"and":[47,106,167,181,199,203,218],"thus":[48],"cannot":[49],"reflect":[50],"nuanced":[52],"performance":[53,82],"these":[55,91,209],"specific":[56,136],"aspects.On":[57],"other":[59],"they":[61],"limited":[63],"diversity":[66,102],"task":[68,107,153],"formats":[69],"only":[71],"multi-choice":[72],"QA),":[73],"which":[74,99,139],"hinders":[75],"understanding":[77],"how":[79],"may":[83],"vary":[84],"across":[85],"types":[87],"tasks.Motivated":[89],"by":[90],"two":[92,115],"problems,":[93],"we":[94,113,122,155,191],"propose":[95,156],"TempCompass":[97],"benchmark,":[98],"introduces":[100],"formats.To":[108],"collect":[109,151],"high-quality":[110],"test":[111],"data,":[112],"devise":[114],"novel":[116],"strategies:":[117],"(1)":[118],"In":[119],"collection,":[121],"construct":[123],"conflicting":[124],"videos":[125],"that":[126,208],"share":[127],"same":[129],"static":[130],"content":[131],"but":[132],"differ":[133],"aspect,":[138],"prevents":[140],"LLMs":[142,198],"from":[143,186],"leveraging":[144],"single-frame":[145],"bias":[146],"or":[147],"priors.(2)":[149],"To":[150],"instructions,":[154],"paradigm":[158],"where":[159],"humans":[160],"first":[161],"annotate":[162],"meta-information":[163],"for":[164],"then":[168],"an":[169,176],"LLM":[170],"generates":[171],"instruction.We":[173],"also":[174],"design":[175],"LLM-based":[177],"approach":[178],"automatically":[180],"accurately":[182],"evaluate":[183,193],"responses":[185],"LLMs.Based":[188],"TempCompass,":[190],"comprehensively":[192],"9":[194],"state-of-the-art":[195],"(SOTA)":[196],"3":[200],"Image":[201],"LLMs,":[202],"reveal":[204],"discerning":[206],"fact":[207],"exhibit":[211],"notably":[212],"poor":[213],"ability.The":[216],"data":[217],"evaluation":[219],"code":[220],"available":[222],"at":[223],"https:":[224],"//github.com/llyx97/TempCompass.":[225]},"counts_by_year":[{"year":2026,"cited_by_count":107},{"year":2025,"cited_by_count":150},{"year":2024,"cited_by_count":28}],"updated_date":"2026-11-10T11:39:43.021295","created_date":"2024-09-21T00:00:00"}