{"id":"https://openalex.org/W7154085730","doi":"https://doi.org/10.1145/3772318.3790922","title":"GazeCoT: Unleashing Social Intelligence in Multimodal LLMs With Gaze-Informed Chain-of-Thought Reasoning","display_name":"GazeCoT: Unleashing Social Intelligence in Multimodal LLMs With Gaze-Informed Chain-of-Thought Reasoning","publication_year":2026,"publication_date":"2026-04-13","ids":{"openalex":"https://openalex.org/W7154085730","doi":"https://doi.org/10.1145/3772318.3790922"},"language":null,"primary_location":{"id":"doi:10.1145/3772318.3790922","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3772318.3790922","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2026 CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3772318.3790922","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5119965679","display_name":"Zhoutong Ye","orcid":null},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Zhoutong Ye","raw_affiliation_strings":["Department of Computer Science and Technology, BNRist, Tsinghua University, Beijing, China"],"raw_orcid":"https://orcid.org/0009-0009-9929-4734","affiliations":[{"raw_affiliation_string":"Department of Computer Science and Technology, BNRist, Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Xutong Wang","orcid":"https://orcid.org/0009-0009-6461-5446"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xutong Wang","raw_affiliation_strings":["Department of Computer Science and Technology, Tsinghua University, Beijing, China"],"raw_orcid":"https://orcid.org/0009-0009-6461-5446","affiliations":[{"raw_affiliation_string":"Department of Computer Science and Technology, Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5015340706","display_name":"Chengwen Zhang","orcid":null},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Chengwen Zhang","raw_affiliation_strings":["Department of Computer Science and Technology, Tsinghua University, Beijing, China"],"raw_orcid":"https://orcid.org/0009-0003-4285-7192","affiliations":[{"raw_affiliation_string":"Department of Computer Science and Technology, Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5088978867","display_name":"Ruiwen Zhang","orcid":null},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Ruiwen Zhang","raw_affiliation_strings":["Academy of Arts & Design, Tsinghua University, Beijing, China"],"raw_orcid":"https://orcid.org/0009-0005-1146-9474","affiliations":[{"raw_affiliation_string":"Academy of Arts & Design, Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5045587504","display_name":"Mingze Sun","orcid":"https://orcid.org/0000-0002-4205-7182"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Mingze Sun","raw_affiliation_strings":["Department of Computer Science and Technology, Tsinghua University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-4205-7182","affiliations":[{"raw_affiliation_string":"Department of Computer Science and Technology, Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Qinwei Li","orcid":"https://orcid.org/0009-0000-8132-9091"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Qinwei Li","raw_affiliation_strings":["Department of Computer Science and Technology, Tsinghua University, Beijing, China"],"raw_orcid":"https://orcid.org/0009-0000-8132-9091","affiliations":[{"raw_affiliation_string":"Department of Computer Science and Technology, Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Chun Yu","orcid":"https://orcid.org/0000-0003-2591-7993"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Chun Yu","raw_affiliation_strings":["Department of Computer Science and Technology, BNRist, College of AI, Tsinghua University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0003-2591-7993","affiliations":[{"raw_affiliation_string":"Department of Computer Science and Technology, BNRist, College of AI, Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"last","author":{"id":null,"display_name":"Yuanchun Shi","orcid":"https://orcid.org/0000-0003-2273-6927"},"institutions":[{"id":"https://openalex.org/I116265982","display_name":"Qinghai University","ror":"https://ror.org/05h33bt13","country_code":"CN","type":"education","lineage":["https://openalex.org/I116265982"]},{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yuanchun Shi","raw_affiliation_strings":["Department of Computer Science and Technology, BNRist, Tsinghua University, Beijing, China and Qinghai University, Xining, China"],"raw_orcid":"https://orcid.org/0000-0003-2273-6927","affiliations":[{"raw_affiliation_string":"Department of Computer Science and Technology, BNRist, Tsinghua University, Beijing, China and Qinghai University, Xining, China","institution_ids":["https://openalex.org/I99065089","https://openalex.org/I116265982"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":8,"corresponding_author_ids":["https://openalex.org/A5119965679"],"corresponding_institution_ids":["https://openalex.org/I99065089"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":{"value":0.83656933,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"24"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11707","display_name":"Gaze Tracking and Assistive Technology","score":0.900600016117096,"subfield":{"id":"https://openalex.org/subfields/1709","display_name":"Human-Computer Interaction"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11707","display_name":"Gaze Tracking and Assistive Technology","score":0.900600016117096,"subfield":{"id":"https://openalex.org/subfields/1709","display_name":"Human-Computer Interaction"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.015399999916553497,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10429","display_name":"EEG and Brain-Computer Interfaces","score":0.013199999928474426,"subfield":{"id":"https://openalex.org/subfields/2805","display_name":"Cognitive Neuroscience"},"field":{"id":"https://openalex.org/fields/28","display_name":"Neuroscience"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/gaze","display_name":"Gaze","score":0.6970999836921692},{"id":"https://openalex.org/keywords/perception","display_name":"Perception","score":0.5680000185966492},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.5048999786376953},{"id":"https://openalex.org/keywords/social-intelligence","display_name":"Social intelligence","score":0.4339999854564667},{"id":"https://openalex.org/keywords/modality","display_name":"Modality (human\u2013computer interaction)","score":0.43299999833106995},{"id":"https://openalex.org/keywords/social-environment","display_name":"Social environment","score":0.40549999475479126},{"id":"https://openalex.org/keywords/pipeline","display_name":"Pipeline (software)","score":0.36640000343322754},{"id":"https://openalex.org/keywords/social-perception","display_name":"Social perception","score":0.36309999227523804}],"concepts":[{"id":"https://openalex.org/C2779916870","wikidata":"https://www.wikidata.org/wiki/Q14467155","display_name":"Gaze","level":2,"score":0.6970999836921692},{"id":"https://openalex.org/C26760741","wikidata":"https://www.wikidata.org/wiki/Q160402","display_name":"Perception","level":2,"score":0.5680000185966492},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.5048999786376953},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.4968000054359436},{"id":"https://openalex.org/C45801056","wikidata":"https://www.wikidata.org/wiki/Q2273844","display_name":"Social intelligence","level":2,"score":0.4339999854564667},{"id":"https://openalex.org/C2780226545","wikidata":"https://www.wikidata.org/wiki/Q6888030","display_name":"Modality (human\u2013computer interaction)","level":2,"score":0.43299999833106995},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.412200003862381},{"id":"https://openalex.org/C110379092","wikidata":"https://www.wikidata.org/wiki/Q1571836","display_name":"Social environment","level":2,"score":0.40549999475479126},{"id":"https://openalex.org/C43521106","wikidata":"https://www.wikidata.org/wiki/Q2165493","display_name":"Pipeline (software)","level":2,"score":0.36640000343322754},{"id":"https://openalex.org/C129484327","wikidata":"https://www.wikidata.org/wiki/Q177603","display_name":"Social perception","level":3,"score":0.36309999227523804},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.3605000078678131},{"id":"https://openalex.org/C56461940","wikidata":"https://www.wikidata.org/wiki/Q970687","display_name":"Eye tracking","level":2,"score":0.3450999855995178},{"id":"https://openalex.org/C10090317","wikidata":"https://www.wikidata.org/wiki/Q7551030","display_name":"Social cue","level":2,"score":0.3319000005722046},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.3276999890804291},{"id":"https://openalex.org/C171268870","wikidata":"https://www.wikidata.org/wiki/Q1486676","display_name":"GRASP","level":2,"score":0.32659998536109924},{"id":"https://openalex.org/C86658582","wikidata":"https://www.wikidata.org/wiki/Q1432778","display_name":"Social cognition","level":3,"score":0.3257000148296356},{"id":"https://openalex.org/C130064352","wikidata":"https://www.wikidata.org/wiki/Q853725","display_name":"Social relation","level":2,"score":0.31310001015663147},{"id":"https://openalex.org/C74216064","wikidata":"https://www.wikidata.org/wiki/Q615684","display_name":"Social computing","level":3,"score":0.31290000677108765},{"id":"https://openalex.org/C105409693","wikidata":"https://www.wikidata.org/wiki/Q5937824","display_name":"Human intelligence","level":2,"score":0.3059999942779541},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.26669999957084656},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.26489999890327454},{"id":"https://openalex.org/C188147891","wikidata":"https://www.wikidata.org/wiki/Q147638","display_name":"Cognitive science","level":1,"score":0.2648000121116638},{"id":"https://openalex.org/C2775936607","wikidata":"https://www.wikidata.org/wiki/Q466845","display_name":"Tracking (education)","level":2,"score":0.26190000772476196},{"id":"https://openalex.org/C2777508537","wikidata":"https://www.wikidata.org/wiki/Q7936620","display_name":"Visual reasoning","level":2,"score":0.2596000134944916},{"id":"https://openalex.org/C139475501","wikidata":"https://www.wikidata.org/wiki/Q17081866","display_name":"Social heuristics","level":4,"score":0.25839999318122864}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3772318.3790922","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3772318.3790922","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2026 CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3772318.3790922","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3772318.3790922","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2026 CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"},"sustainable_development_goals":[{"score":0.40868067741394043,"display_name":"Reduced inequalities","id":"https://metadata.un.org/sdg/10"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":138,"referenced_works":["https://openalex.org/W70857234","https://openalex.org/W1964809999","https://openalex.org/W1965493816","https://openalex.org/W1997326154","https://openalex.org/W2029003128","https://openalex.org/W2051643022","https://openalex.org/W2059216172","https://openalex.org/W2077324971","https://openalex.org/W2095535364","https://openalex.org/W2101148991","https://openalex.org/W2121478585","https://openalex.org/W2135907989","https://openalex.org/W2137354906","https://openalex.org/W2147734093","https://openalex.org/W2150839178","https://openalex.org/W2173179171","https://openalex.org/W2296512045","https://openalex.org/W2298851760","https://openalex.org/W2525311127","https://openalex.org/W2566422630","https://openalex.org/W2594627455","https://openalex.org/W2759653627","https://openalex.org/W2798558457","https://openalex.org/W2898686814","https://openalex.org/W2910307394","https://openalex.org/W2962701817","https://openalex.org/W2962762462","https://openalex.org/W2963095307","https://openalex.org/W2964290048","https://openalex.org/W2970062726","https://openalex.org/W3016099278","https://openalex.org/W3023585286","https://openalex.org/W3035735638","https://openalex.org/W3126664801","https://openalex.org/W3154013571","https://openalex.org/W3160114030","https://openalex.org/W3163411042","https://openalex.org/W3164939684","https://openalex.org/W3174359952","https://openalex.org/W3177001938","https://openalex.org/W3188642398","https://openalex.org/W3190266158","https://openalex.org/W3197864072","https://openalex.org/W3210874941","https://openalex.org/W4200168337","https://openalex.org/W4205190925","https://openalex.org/W4211153864","https://openalex.org/W4213426744","https://openalex.org/W4220846842","https://openalex.org/W4243342770","https://openalex.org/W4292793996","https://openalex.org/W4309618066","https://openalex.org/W4309619902","https://openalex.org/W4312563428","https://openalex.org/W4312567381","https://openalex.org/W4319321305","https://openalex.org/W4323911378","https://openalex.org/W4366548417","https://openalex.org/W4379985979","https://openalex.org/W4382457317","https://openalex.org/W4384663241","https://openalex.org/W4387250680","https://openalex.org/W4387618732","https://openalex.org/W4390873481","https://openalex.org/W4390873964","https://openalex.org/W4390874396","https://openalex.org/W4390874575","https://openalex.org/W4392353733","https://openalex.org/W4392658184","https://openalex.org/W4396832189","https://openalex.org/W4396832347","https://openalex.org/W4396832611","https://openalex.org/W4396833254","https://openalex.org/W4396938940","https://openalex.org/W4399072199","https://openalex.org/W4400303997","https://openalex.org/W4402427038","https://openalex.org/W4402551159","https://openalex.org/W4402670262","https://openalex.org/W4402671125","https://openalex.org/W4402716166","https://openalex.org/W4402753807","https://openalex.org/W4402916798","https://openalex.org/W4402968400","https://openalex.org/W4403071981","https://openalex.org/W4403333954","https://openalex.org/W4403334451","https://openalex.org/W4403555634","https://openalex.org/W4403577434","https://openalex.org/W4403717124","https://openalex.org/W4403780570","https://openalex.org/W4403908252","https://openalex.org/W4404199184","https://openalex.org/W4404199570","https://openalex.org/W4404294163","https://openalex.org/W4404356490","https://openalex.org/W4404781649","https://openalex.org/W4405022833","https://openalex.org/W4406890873","https://openalex.org/W4408345794","https://openalex.org/W4408954391","https://openalex.org/W4409367601","https://openalex.org/W4409720608","https://openalex.org/W4409720872","https://openalex.org/W4409736015","https://openalex.org/W4409745910","https://openalex.org/W4409746179","https://openalex.org/W4409746813","https://openalex.org/W4409884218","https://openalex.org/W4409884738","https://openalex.org/W4409885603","https://openalex.org/W4409906271","https://openalex.org/W4410701788","https://openalex.org/W4411120062","https://openalex.org/W4412035737","https://openalex.org/W4412045075","https://openalex.org/W4412158322","https://openalex.org/W4412376973","https://openalex.org/W4412664050","https://openalex.org/W4412888616","https://openalex.org/W4412895952","https://openalex.org/W4413144646","https://openalex.org/W4413147382","https://openalex.org/W4413147571","https://openalex.org/W4414564720","https://openalex.org/W4414983863","https://openalex.org/W4415538710","https://openalex.org/W4415795406","https://openalex.org/W4415798483","https://openalex.org/W4415799075","https://openalex.org/W4415799167","https://openalex.org/W7084077715","https://openalex.org/W7084152796","https://openalex.org/W7103748293","https://openalex.org/W7103749886","https://openalex.org/W7103750215","https://openalex.org/W7103751227","https://openalex.org/W7120088840"],"related_works":[],"abstract_inverted_index":{"Social":[0],"intelligence":[1,100],"is":[2,31,75],"vital":[3],"for":[4],"effective":[5],"human-AI":[6],"interaction.":[7],"While":[8],"LLMs":[9,48],"demonstrate":[10],"strong":[11],"text-based":[12],"social":[13,26,36,90,99,126,129],"intelligence,":[14],"the":[15,22,32,64],"vision":[16],"modality":[17],"remains":[18],"challenging":[19,110],"due":[20],"to":[21,60,87],"presence":[23],"of":[24,35,66],"non-verbal":[25],"cues.":[27],"For":[28],"example,":[29],"gaze":[30,57,73,103],"primary":[33],"conveyor":[34],"attention,":[37],"yet":[38],"it":[39],"cannot":[40],"be":[41],"accurately":[42],"perceived":[43,119],"and":[44,79,121,128],"understood":[45],"by":[46,101,123],"multimodal":[47],"(MLLMs).":[49],"Therefore,":[50],"we":[51],"propose":[52],"GazeCoT,":[53,137],"a":[54,84,109,138],"pipeline":[55],"using":[56],"estimation":[58],"models":[59],"provide":[61],"MLLMs":[62],"with":[63,131],"attention":[65],"people":[67],"in":[68,108],"images":[69],"or":[70],"videos.":[71],"The":[72],"information":[74],"provided":[76],"as":[77],"visual":[78],"text":[80],"prompts":[81],"compiled":[82],"into":[83],"structured":[85],"context":[86],"support":[88],"MLLM":[89,125],"reasoning.":[91],"Benchmark":[92],"evaluation":[93],"confirms":[94],"that":[95,116,136],"GazeCoT":[96,117],"enhances":[97],"MLLMs\u2019":[98],"improving":[102],"perception.":[104],"A":[105],"user":[106],"study":[107],"application":[111],"involving":[112],"parent-child":[113],"interactions":[114],"demonstrates":[115],"improves":[118],"explainability":[120],"trustworthiness":[122],"aligning":[124],"perception":[127],"reasoning":[130],"human":[132],"norms.":[133],"We":[134],"hope":[135],"versatile":[139],"plug-and-play":[140],"pipeline,":[141],"can":[142],"enable":[143],"socially":[144],"aware,":[145],"MLLM-based":[146],"HCI":[147],"applications.":[148]},"counts_by_year":[],"updated_date":"2026-04-25T08:17:42.794288","created_date":"2026-04-14T00:00:00"}