{"id":"https://openalex.org/W4403792038","doi":"https://doi.org/10.1145/3664647.3680810","title":"Hypergraph Multi-modal Large Language Model: Exploiting EEG and Eye-tracking Modalities to Evaluate Heterogeneous Responses for Video Understanding","display_name":"Hypergraph Multi-modal Large Language Model: Exploiting EEG and Eye-tracking Modalities to Evaluate Heterogeneous Responses for Video Understanding","publication_year":2024,"publication_date":"2024-10-26","ids":{"openalex":"https://openalex.org/W4403792038","doi":"https://doi.org/10.1145/3664647.3680810"},"language":"en","primary_location":{"id":"doi:10.1145/3664647.3680810","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3664647.3680810","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 32nd ACM International Conference on Multimedia","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5082743444","display_name":"Minghui Wu","orcid":"https://orcid.org/0009-0001-4577-3002"},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Minghui Wu","raw_affiliation_strings":["Mininglamp Technology & Peking University, Beijing, China"],"affiliations":[{"raw_affiliation_string":"Mininglamp Technology & Peking University, Beijing, China","institution_ids":["https://openalex.org/I20231570"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101463817","display_name":"Chenxu Zhao","orcid":"https://orcid.org/0000-0003-4044-5701"},"institutions":[{"id":"https://openalex.org/I4401726980","display_name":"Mininglamp (China)","ror":"https://ror.org/04tb90x61","country_code":"CN","type":"company","lineage":["https://openalex.org/I4401726980"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Chenxu Zhao","raw_affiliation_strings":["Mininglamp Technology, Beijing, China"],"affiliations":[{"raw_affiliation_string":"Mininglamp Technology, Beijing, China","institution_ids":["https://openalex.org/I4401726980"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5042083601","display_name":"Anyang Su","orcid":"https://orcid.org/0000-0002-0085-3527"},"institutions":[{"id":"https://openalex.org/I4401726980","display_name":"Mininglamp (China)","ror":"https://ror.org/04tb90x61","country_code":"CN","type":"company","lineage":["https://openalex.org/I4401726980"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Anyang Su","raw_affiliation_strings":["Mininglamp Technology, Beijing, China"],"affiliations":[{"raw_affiliation_string":"Mininglamp Technology, Beijing, China","institution_ids":["https://openalex.org/I4401726980"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5031208819","display_name":"Donglin Di","orcid":"https://orcid.org/0000-0002-2270-3378"},"institutions":[{"id":"https://openalex.org/I4210100255","display_name":"Beijing Academy of Artificial Intelligence","ror":"https://ror.org/016a74861","country_code":"CN","type":"other","lineage":["https://openalex.org/I4210100255"]},{"id":"https://openalex.org/I4391012619","display_name":"Shanghai Artificial Intelligence Laboratory","ror":"https://ror.org/03wkvpx79","country_code":null,"type":"facility","lineage":["https://openalex.org/I4391012619"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Donglin Di","raw_affiliation_strings":["Shanghai Artificial Intelligence Lab, Shanghai, China"],"affiliations":[{"raw_affiliation_string":"Shanghai Artificial Intelligence Lab, Shanghai, China","institution_ids":["https://openalex.org/I4210100255","https://openalex.org/I4391012619"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5041718044","display_name":"Tianyu Fu","orcid":"https://orcid.org/0009-0007-0061-8184"},"institutions":[{"id":"https://openalex.org/I4210100255","display_name":"Beijing Academy of Artificial Intelligence","ror":"https://ror.org/016a74861","country_code":"CN","type":"other","lineage":["https://openalex.org/I4210100255"]},{"id":"https://openalex.org/I4391012619","display_name":"Shanghai Artificial Intelligence Laboratory","ror":"https://ror.org/03wkvpx79","country_code":null,"type":"facility","lineage":["https://openalex.org/I4391012619"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Tianyu Fu","raw_affiliation_strings":["Shanghai Artificial Intelligence Lab, Shanghai, China"],"affiliations":[{"raw_affiliation_string":"Shanghai Artificial Intelligence Lab, Shanghai, China","institution_ids":["https://openalex.org/I4210100255","https://openalex.org/I4391012619"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5089243309","display_name":"Da An","orcid":null},"institutions":[{"id":"https://openalex.org/I4210100255","display_name":"Beijing Academy of Artificial Intelligence","ror":"https://ror.org/016a74861","country_code":"CN","type":"other","lineage":["https://openalex.org/I4210100255"]},{"id":"https://openalex.org/I4391012619","display_name":"Shanghai Artificial Intelligence Laboratory","ror":"https://ror.org/03wkvpx79","country_code":null,"type":"facility","lineage":["https://openalex.org/I4391012619"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Da An","raw_affiliation_strings":["Shanghai Artificial Intelligence Lab, Shanghai, China"],"affiliations":[{"raw_affiliation_string":"Shanghai Artificial Intelligence Lab, Shanghai, China","institution_ids":["https://openalex.org/I4210100255","https://openalex.org/I4391012619"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5109470789","display_name":"Min He","orcid":"https://orcid.org/0009-0001-1680-5239"},"institutions":[{"id":"https://openalex.org/I4401726980","display_name":"Mininglamp (China)","ror":"https://ror.org/04tb90x61","country_code":"CN","type":"company","lineage":["https://openalex.org/I4401726980"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Min He","raw_affiliation_strings":["MiningLamp Technology, Beijing, China"],"affiliations":[{"raw_affiliation_string":"MiningLamp Technology, Beijing, China","institution_ids":["https://openalex.org/I4401726980"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5045853744","display_name":"Ya Gao","orcid":"https://orcid.org/0000-0002-9785-922X"},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Ya Gao","raw_affiliation_strings":["Peking University, Beijing, China"],"affiliations":[{"raw_affiliation_string":"Peking University, Beijing, China","institution_ids":["https://openalex.org/I20231570"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100762316","display_name":"Meng Ma","orcid":"https://orcid.org/0000-0002-1963-2513"},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Meng Ma","raw_affiliation_strings":["Peking University, Beijing, China"],"affiliations":[{"raw_affiliation_string":"Peking University, Beijing, China","institution_ids":["https://openalex.org/I20231570"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5106355302","display_name":"Kun Yan","orcid":"https://orcid.org/0000-0002-1234-6119"},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Kun Yan","raw_affiliation_strings":["Peking University, Beijing, China"],"affiliations":[{"raw_affiliation_string":"Peking University, Beijing, China","institution_ids":["https://openalex.org/I20231570"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100338689","display_name":"Ping Wang","orcid":"https://orcid.org/0000-0002-8854-2079"},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Ping Wang","raw_affiliation_strings":["Peking University, Beijing, China"],"affiliations":[{"raw_affiliation_string":"Peking University, Beijing, China","institution_ids":["https://openalex.org/I20231570"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":11,"corresponding_author_ids":["https://openalex.org/A5082743444"],"corresponding_institution_ids":["https://openalex.org/I20231570"],"apc_list":null,"apc_paid":null,"fwci":1.2464,"has_fulltext":false,"cited_by_count":5,"citation_normalized_percentile":{"value":0.81410539,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":96,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"7316","last_page":"7325"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10812","display_name":"Human Pose and Action Recognition","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10812","display_name":"Human Pose and Action Recognition","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11512","display_name":"Anomaly Detection Techniques and Applications","score":0.9825999736785889,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/modalities","display_name":"Modalities","score":0.8115501999855042},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7912243008613586},{"id":"https://openalex.org/keywords/modal","display_name":"Modal","score":0.6248639822006226},{"id":"https://openalex.org/keywords/eye-tracking","display_name":"Eye tracking","score":0.5749574899673462},{"id":"https://openalex.org/keywords/electroencephalography","display_name":"Electroencephalography","score":0.5055050849914551},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.46544086933135986},{"id":"https://openalex.org/keywords/hypergraph","display_name":"Hypergraph","score":0.45213478803634644},{"id":"https://openalex.org/keywords/tracking","display_name":"Tracking (education)","score":0.43529051542282104},{"id":"https://openalex.org/keywords/computer-vision","display_name":"Computer vision","score":0.4171692430973053},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.08252084255218506},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.0690203607082367}],"concepts":[{"id":"https://openalex.org/C2779903281","wikidata":"https://www.wikidata.org/wiki/Q6888026","display_name":"Modalities","level":2,"score":0.8115501999855042},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7912243008613586},{"id":"https://openalex.org/C71139939","wikidata":"https://www.wikidata.org/wiki/Q910194","display_name":"Modal","level":2,"score":0.6248639822006226},{"id":"https://openalex.org/C56461940","wikidata":"https://www.wikidata.org/wiki/Q970687","display_name":"Eye tracking","level":2,"score":0.5749574899673462},{"id":"https://openalex.org/C522805319","wikidata":"https://www.wikidata.org/wiki/Q179965","display_name":"Electroencephalography","level":2,"score":0.5055050849914551},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.46544086933135986},{"id":"https://openalex.org/C2781221856","wikidata":"https://www.wikidata.org/wiki/Q840247","display_name":"Hypergraph","level":2,"score":0.45213478803634644},{"id":"https://openalex.org/C2775936607","wikidata":"https://www.wikidata.org/wiki/Q466845","display_name":"Tracking (education)","level":2,"score":0.43529051542282104},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.4171692430973053},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.08252084255218506},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0690203607082367},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.0},{"id":"https://openalex.org/C19417346","wikidata":"https://www.wikidata.org/wiki/Q7922","display_name":"Pedagogy","level":1,"score":0.0},{"id":"https://openalex.org/C118552586","wikidata":"https://www.wikidata.org/wiki/Q7867","display_name":"Psychiatry","level":1,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C36289849","wikidata":"https://www.wikidata.org/wiki/Q34749","display_name":"Social science","level":1,"score":0.0},{"id":"https://openalex.org/C188027245","wikidata":"https://www.wikidata.org/wiki/Q750446","display_name":"Polymer chemistry","level":1,"score":0.0},{"id":"https://openalex.org/C118615104","wikidata":"https://www.wikidata.org/wiki/Q121416","display_name":"Discrete mathematics","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3664647.3680810","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3664647.3680810","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 32nd ACM International Conference on Multimedia","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":44,"referenced_works":["https://openalex.org/W1556771345","https://openalex.org/W1947481528","https://openalex.org/W1954735160","https://openalex.org/W2014854862","https://openalex.org/W2051090063","https://openalex.org/W2054747732","https://openalex.org/W2068078373","https://openalex.org/W2149628368","https://openalex.org/W2185525576","https://openalex.org/W2288381231","https://openalex.org/W2342662179","https://openalex.org/W2507009361","https://openalex.org/W2606982687","https://openalex.org/W2766402183","https://openalex.org/W2790404832","https://openalex.org/W2796862151","https://openalex.org/W2889055034","https://openalex.org/W2892880750","https://openalex.org/W2922118332","https://openalex.org/W2929974724","https://openalex.org/W2963355311","https://openalex.org/W2963524571","https://openalex.org/W2964216549","https://openalex.org/W2964220823","https://openalex.org/W2967918518","https://openalex.org/W2988743243","https://openalex.org/W2990503944","https://openalex.org/W3014163061","https://openalex.org/W3090449556","https://openalex.org/W3100481960","https://openalex.org/W3104355095","https://openalex.org/W3126721948","https://openalex.org/W3127632369","https://openalex.org/W3156356077","https://openalex.org/W3189373506","https://openalex.org/W3210279979","https://openalex.org/W4214612132","https://openalex.org/W4295774968","https://openalex.org/W4304091755","https://openalex.org/W4378364840","https://openalex.org/W4383988883","https://openalex.org/W4386071707","https://openalex.org/W4386076656","https://openalex.org/W6786266798"],"related_works":["https://openalex.org/W4376608589","https://openalex.org/W1537073411","https://openalex.org/W1948107826","https://openalex.org/W3138003926","https://openalex.org/W1630514295","https://openalex.org/W4300037846","https://openalex.org/W4288275998","https://openalex.org/W2963081352","https://openalex.org/W4301143707","https://openalex.org/W2952745240"],"abstract_inverted_index":{"Understanding":[0],"of":[1,29,46,125,128,193],"video":[2,108,129,154],"creativity":[3],"and":[4,15,22,34,48,56,66,98,117,121,157,168,183,198],"content":[5,55,130],"often":[6],"varies":[7],"among":[8,131,151],"individuals,":[9],"with":[10,50,135],"differences":[11],"in":[12,31,95],"focal":[13],"points":[14],"cognitive":[16,126],"levels":[17],"across":[18,165],"different":[19,102,132,152,172],"ages,":[20],"experiences,":[21],"genders.":[23],"There":[24],"is":[25],"currently":[26],"a":[27,43,81,140],"lack":[28],"research":[30],"this":[32,111],"area,":[33],"most":[35],"existing":[36],"benchmarks":[37,189],"suffer":[38],"from":[39,101],"several":[40],"drawbacks:":[41],"1)":[42],"limited":[44],"number":[45],"modalities":[47,167,173],"answers":[49],"restrictive":[51],"length;":[52],"2)":[53],"the":[54,59,74,123,136,149,191],"scenarios":[57],"within":[58],"videos":[60],"are":[61,69,200],"excessively":[62],"monotonous,":[63],"transmitting":[64],"allegories":[65],"emotions":[67],"that":[68],"overly":[70],"simplistic.":[71],"To":[72],"bridge":[73,162],"gap":[75],"to":[76,119,147,174],"real-world":[77],"applications,":[78],"we":[79,91,114,138],"introduce":[80],"large-scale":[82],"Video":[83],"Subjective":[84],"Multi-modal":[85,142],"Evaluation":[86],"dataset,":[87,113,137],"namely":[88],"Video-SME.":[89],"Specifically,":[90],"collected":[92],"real":[93],"changes":[94],"Electroencephalographic":[96],"(EEG)":[97],"eye-tracking":[99,158],"regions":[100],"demographics":[103],"while":[104],"they":[105],"viewed":[106],"identical":[107],"content.":[109],"Utilizing":[110],"multi-modal":[112],"developed":[115],"tasks":[116],"protocols":[118],"analyze":[120],"evaluate":[122],"extent":[124],"understanding":[127],"users.":[133],"Along":[134],"designed":[139],"Hypergraph":[141],"Large":[143],"Language":[144],"Model":[145],"(HMLLM)":[146],"explore":[148],"associations":[150],"demographics,":[153],"elements,":[155],"EEG":[156],"indicators.":[159],"HMLLM":[160],"could":[161],"semantic":[163],"gaps":[164],"rich":[166],"integrate":[169],"information":[170],"beyond":[171],"perform":[175],"logical":[176],"reasoning.":[177],"Extensive":[178],"experimental":[179],"evaluations":[180],"on":[181],"Video-SME":[182],"other":[184],"additional":[185],"video-based":[186],"generative":[187],"performance":[188],"demonstrate":[190],"effectiveness":[192],"our":[194],"method.":[195],"The":[196],"code":[197],"dataset":[199],"available":[201],"at":[202],"https://github.com/mininglamp-MLLM/HMLLM":[203]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":4}],"updated_date":"2026-03-27T05:58:40.876381","created_date":"2025-10-10T00:00:00"}