{"id":"https://openalex.org/W7225274407","doi":"https://doi.org/10.48550/arxiv.2611.05181","title":"What Does the Model See? Representation Validity in Multimodal LLM-Based Video Analysis","display_name":"What Does the Model See? Representation Validity in Multimodal LLM-Based Video Analysis","publication_year":2026,"publication_date":"2026-11-03","ids":{"openalex":"https://openalex.org/W7225274407","doi":"https://doi.org/10.48550/arxiv.2611.05181"},"language":"en","primary_location":{"id":"doi:10.48550/arxiv.2611.05181","doi":"https://doi.org/10.48550/arxiv.2611.05181","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2611.05181","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2611.05181","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5034464160","display_name":"Christian Pipal","orcid":"https://orcid.org/0000-0002-5395-2035","observed_orcids":["https://orcid.org/0000-0002-5395-2035"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Pipal, Christian","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5014527442","display_name":"Eva-Maria Vogel","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Vogel, Eva-Maria","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5017612757","display_name":"Morgan Wack","orcid":"https://orcid.org/0000-0002-7769-5993","observed_orcids":["https://orcid.org/0000-0002-7769-5993"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wack, Morgan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13910","display_name":"Computational and Text Analysis Methods","score":0.9179999828338623,"subfield":{"id":"https://openalex.org/subfields/3300","display_name":"General Social Sciences"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T13910","display_name":"Computational and Text Analysis Methods","score":0.9179999828338623,"subfield":{"id":"https://openalex.org/subfields/3300","display_name":"General Social Sciences"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.06930000334978104,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10664","display_name":"Sentiment Analysis and Opinion Mining","score":0.0031999999191612005,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/multimodal-large-language-models","display_name":"multimodal large language models","score":0.9470000267028809},{"id":"https://openalex.org/keywords/video-analysis","display_name":"video analysis","score":0.8679999709129333},{"id":"https://openalex.org/keywords/input-representation","display_name":"input representation","score":0.5080000162124634},{"id":"https://openalex.org/keywords/audio-input","display_name":"audio input","score":0.4129999876022339},{"id":"https://openalex.org/keywords/news-content-analysis","display_name":"news content analysis","score":0.5009999871253967},{"id":"https://openalex.org/keywords/human-annotation-comparison","display_name":"human annotation comparison","score":0.45500001311302185},{"id":"https://openalex.org/keywords/regression-parameter-estimation","display_name":"regression coefficient estimation","score":0.5740000009536743},{"id":"https://openalex.org/keywords/measurement-validity","display_name":"measurement validity","score":0.4790000021457672}],"concepts":[{"id":"https://openalex.org/C2776359362","wikidata":"https://www.wikidata.org/wiki/Q2145286","display_name":"Representation (politics)","level":3,"score":0.7512999773025513},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6966000199317932},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.6259999871253967},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6011000275611877},{"id":"https://openalex.org/C2780801425","wikidata":"https://www.wikidata.org/wiki/Q5164392","display_name":"Construct (python library)","level":2,"score":0.5717999935150146},{"id":"https://openalex.org/C83546350","wikidata":"https://www.wikidata.org/wiki/Q1139051","display_name":"Regression","level":2,"score":0.439300000667572},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.42590001225471497},{"id":"https://openalex.org/C152877465","wikidata":"https://www.wikidata.org/wiki/Q208042","display_name":"Regression analysis","level":2,"score":0.39259999990463257},{"id":"https://openalex.org/C61797465","wikidata":"https://www.wikidata.org/wiki/Q1188986","display_name":"Term (time)","level":2,"score":0.3804999887943268},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.3474000096321106},{"id":"https://openalex.org/C2776035688","wikidata":"https://www.wikidata.org/wiki/Q1606558","display_name":"Affect (linguistics)","level":2,"score":0.3203999996185303},{"id":"https://openalex.org/C49453240","wikidata":"https://www.wikidata.org/wiki/Q1592163","display_name":"Construct validity","level":3,"score":0.2962999939918518},{"id":"https://openalex.org/C2776502983","wikidata":"https://www.wikidata.org/wiki/Q690182","display_name":"Contrast (vision)","level":2,"score":0.28040000796318054},{"id":"https://openalex.org/C126042441","wikidata":"https://www.wikidata.org/wiki/Q1324888","display_name":"Frame (networking)","level":2,"score":0.2791000008583069},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.2718999981880188},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.26489999890327454},{"id":"https://openalex.org/C2780586882","wikidata":"https://www.wikidata.org/wiki/Q7520643","display_name":"Simple (philosophy)","level":2,"score":0.25290000438690186}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2611.05181","doi":"https://doi.org/10.48550/arxiv.2611.05181","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2611.05181","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2611.05181","doi":"https://doi.org/10.48550/arxiv.2611.05181","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2611.05181","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4413146669","https://openalex.org/W4411267778","https://openalex.org/W4415269889","https://openalex.org/W4409720450","https://openalex.org/W4416075179","https://openalex.org/W4417073196","https://openalex.org/W7163525532","https://openalex.org/W7128772733","https://openalex.org/W7164001219","https://openalex.org/W3143239048"],"abstract_inverted_index":{"Multimodal":[0],"large":[1],"language":[2],"models":[3,73,86],"(MLLMs)":[4],"allow":[5],"social":[6],"scientists":[7],"to":[8,22,32,83,110],"analyze":[9],"video":[10,21,57],"at":[11],"scale,":[12],"but":[13],"researchers":[14],"must":[15],"decide":[16],"which":[17],"parts":[18],"of":[19,126,139],"a":[20],"provide":[23],"as":[24],"input.":[25],"We":[26,46],"use":[27],"the":[28,144,160,165,169],"term":[29],"representation":[30,60,118,154],"validity":[31],"examine":[33],"how":[34],"text,":[35],"audio,":[36],"video,":[37],"and":[38,43,54,91,164],"combined":[39],"inputs":[40,96],"affect":[41],"measurement":[42],"subsequent":[44],"analysis.":[45],"compare":[47],"input":[48],"representations":[49],"across":[50,72],"six":[51],"open-weight":[52],"MLLMs":[53],"three":[55,92],"human-annotated":[56],"datasets.":[58],"No":[59],"consistently":[61],"performs":[62],"best.":[63],"On":[64],"seven":[65],"news-related":[66],"constructs,":[67],"native-video":[68],"performance":[69,79],"varies":[70],"little":[71],"(macro-F1":[74],"=":[75],".70-.77),":[76],"while":[77],"extracted-text":[78],"ranges":[80],"from":[81,143,168],".52":[82],".80.":[84],"Three":[85],"perform":[87],"better":[88],"with":[89,93],"text":[90],"video.":[94],"Combining":[95],"rarely":[97],"improves":[98],"performance.":[99],"Higher":[100],"classification":[101],"accuracy":[102],"also":[103],"does":[104],"not":[105],"reliably":[106],"produce":[107],"estimates":[108,141],"closer":[109],"those":[111],"based":[112],"on":[113],"human":[114],"codes.":[115],"The":[116],"highest-F1":[117,135],"best":[119],"reproduces":[120],"human-coded":[121,145],"regression":[122],"coefficients":[123],"in":[124],"17":[125],"42":[127],"comparisons.":[128],"Even":[129],"when":[130],"each":[131],"model":[132],"uses":[133],"its":[134],"deployable":[136],"representation,":[137],"45%":[138],"sample-prevalence":[140],"differ":[142],"benchmark":[146],"by":[147],"more":[148],"than":[149],"ten":[150],"percentage":[151],"points.":[152],"Input":[153],"therefore":[155],"requires":[156],"validation":[157],"for":[158],"both":[159],"construct":[161],"being":[162],"measured":[163],"quantity":[166],"estimated":[167],"resulting":[170],"labels.":[171]},"counts_by_year":[],"updated_date":"2026-11-06T12:05:46.655746","created_date":"2026-11-05T00:00:00"}