{"id":"https://openalex.org/W4417025350","doi":"https://doi.org/10.1145/3756884.3766053","title":"PatchFusionVR: Multitask Prediction of User Gaze, Reaction Time, and Cognitive Load in Virtual Reality from Multimodal Signals","display_name":"PatchFusionVR: Multitask Prediction of User Gaze, Reaction Time, and Cognitive Load in Virtual Reality from Multimodal Signals","publication_year":2025,"publication_date":"2025-11-12","ids":{"openalex":"https://openalex.org/W4417025350","doi":"https://doi.org/10.1145/3756884.3766053"},"language":null,"primary_location":{"id":"doi:10.1145/3756884.3766053","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3756884.3766053","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3756884.3766053","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2025 31st ACM Symposium on Virtual Reality Software and Technology","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3756884.3766053","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5058462342","display_name":"Md. Saidur Rahman Pavel","orcid":"https://orcid.org/0009-0006-0627-4215"},"institutions":[{"id":"https://openalex.org/I172980758","display_name":"Kennesaw State University","ror":"https://ror.org/00jeqjx33","country_code":"US","type":"education","lineage":["https://openalex.org/I172980758"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Md Irfan Pavel","raw_affiliation_strings":["Extended Reality and Intelligence Lab, Kennesaw State University, Marietta, Georgia, USA"],"raw_orcid":"https://orcid.org/0009-0006-0627-4215","affiliations":[{"raw_affiliation_string":"Extended Reality and Intelligence Lab, Kennesaw State University, Marietta, Georgia, USA","institution_ids":["https://openalex.org/I172980758"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5069998359","display_name":"M. Rasel Mahmud","orcid":"https://orcid.org/0000-0003-2094-8192"},"institutions":[{"id":"https://openalex.org/I172980758","display_name":"Kennesaw State University","ror":"https://ror.org/00jeqjx33","country_code":"US","type":"education","lineage":["https://openalex.org/I172980758"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"M Rasel Mahmud","raw_affiliation_strings":["Assistant Professor Computer Science, Kennesaw State University, MARIETTA, Georgia, USA"],"raw_orcid":"https://orcid.org/0000-0003-2094-8192","affiliations":[{"raw_affiliation_string":"Assistant Professor Computer Science, Kennesaw State University, MARIETTA, Georgia, USA","institution_ids":["https://openalex.org/I172980758"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5114353227","display_name":"Jyotirmay Nag Setu","orcid":"https://orcid.org/0009-0004-4672-8758"},"institutions":[{"id":"https://openalex.org/I45438204","display_name":"The University of Texas at San Antonio","ror":"https://ror.org/01kd65564","country_code":"US","type":"education","lineage":["https://openalex.org/I45438204"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jyotirmay Nag Setu","raw_affiliation_strings":["San Antonio Virtual Environments (SAVE) Lab, University of Texas at San Antonio, San Antonio, Texas, USA"],"raw_orcid":"https://orcid.org/0009-0004-4672-8758","affiliations":[{"raw_affiliation_string":"San Antonio Virtual Environments (SAVE) Lab, University of Texas at San Antonio, San Antonio, Texas, USA","institution_ids":["https://openalex.org/I45438204"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5076318084","display_name":"Kevin Desai","orcid":"https://orcid.org/0000-0002-2964-8981"},"institutions":[{"id":"https://openalex.org/I45438204","display_name":"The University of Texas at San Antonio","ror":"https://ror.org/01kd65564","country_code":"US","type":"education","lineage":["https://openalex.org/I45438204"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kevin Desai","raw_affiliation_strings":["Computer Science Department, The University of Texas at San Antonio, San Antonio, Texas, USA"],"raw_orcid":"https://orcid.org/0000-0002-2964-8981","affiliations":[{"raw_affiliation_string":"Computer Science Department, The University of Texas at San Antonio, San Antonio, Texas, USA","institution_ids":["https://openalex.org/I45438204"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5084343243","display_name":"John Quarles","orcid":"https://orcid.org/0000-0002-4790-167X"},"institutions":[{"id":"https://openalex.org/I45438204","display_name":"The University of Texas at San Antonio","ror":"https://ror.org/01kd65564","country_code":"US","type":"education","lineage":["https://openalex.org/I45438204"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"John Quarles","raw_affiliation_strings":["Computer Science, University of Texas at San Antonio, San Antonio, Texas, USA"],"raw_orcid":"https://orcid.org/0000-0002-4790-167X","affiliations":[{"raw_affiliation_string":"Computer Science, University of Texas at San Antonio, San Antonio, Texas, USA","institution_ids":["https://openalex.org/I45438204"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5058462342"],"corresponding_institution_ids":["https://openalex.org/I172980758"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":{"value":0.37128146,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"11"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11707","display_name":"Gaze Tracking and Assistive Technology","score":0.8514000177383423,"subfield":{"id":"https://openalex.org/subfields/1709","display_name":"Human-Computer Interaction"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11707","display_name":"Gaze Tracking and Assistive Technology","score":0.8514000177383423,"subfield":{"id":"https://openalex.org/subfields/1709","display_name":"Human-Computer Interaction"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11605","display_name":"Visual Attention and Saliency Detection","score":0.06440000236034393,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13219","display_name":"Mind wandering and attention","score":0.011800000444054604,"subfield":{"id":"https://openalex.org/subfields/2805","display_name":"Cognitive Neuroscience"},"field":{"id":"https://openalex.org/fields/28","display_name":"Neuroscience"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/cognitive-load","display_name":"Cognitive load","score":0.7642999887466431},{"id":"https://openalex.org/keywords/gaze","display_name":"Gaze","score":0.647599995136261},{"id":"https://openalex.org/keywords/virtual-reality","display_name":"Virtual reality","score":0.5702000260353088},{"id":"https://openalex.org/keywords/cognition","display_name":"Cognition","score":0.4465999901294708},{"id":"https://openalex.org/keywords/autoencoder","display_name":"Autoencoder","score":0.43950000405311584},{"id":"https://openalex.org/keywords/visualization","display_name":"Visualization","score":0.4025999903678894},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.40130001306533813},{"id":"https://openalex.org/keywords/encoder","display_name":"Encoder","score":0.3991999924182892}],"concepts":[{"id":"https://openalex.org/C61641136","wikidata":"https://www.wikidata.org/wiki/Q1107019","display_name":"Cognitive load","level":3,"score":0.7642999887466431},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7488999962806702},{"id":"https://openalex.org/C2779916870","wikidata":"https://www.wikidata.org/wiki/Q14467155","display_name":"Gaze","level":2,"score":0.647599995136261},{"id":"https://openalex.org/C194969405","wikidata":"https://www.wikidata.org/wiki/Q170519","display_name":"Virtual reality","level":2,"score":0.5702000260353088},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.48809999227523804},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.4510999917984009},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.45100000500679016},{"id":"https://openalex.org/C169900460","wikidata":"https://www.wikidata.org/wiki/Q2200417","display_name":"Cognition","level":2,"score":0.4465999901294708},{"id":"https://openalex.org/C101738243","wikidata":"https://www.wikidata.org/wiki/Q786435","display_name":"Autoencoder","level":3,"score":0.43950000405311584},{"id":"https://openalex.org/C36464697","wikidata":"https://www.wikidata.org/wiki/Q451553","display_name":"Visualization","level":2,"score":0.4025999903678894},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.40130001306533813},{"id":"https://openalex.org/C118505674","wikidata":"https://www.wikidata.org/wiki/Q42586063","display_name":"Encoder","level":2,"score":0.3991999924182892},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.34279999136924744},{"id":"https://openalex.org/C127162648","wikidata":"https://www.wikidata.org/wiki/Q16858953","display_name":"Channel (broadcasting)","level":2,"score":0.32739999890327454},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.3057999908924103},{"id":"https://openalex.org/C175154964","wikidata":"https://www.wikidata.org/wiki/Q380077","display_name":"Task analysis","level":3,"score":0.2985999882221222},{"id":"https://openalex.org/C56461940","wikidata":"https://www.wikidata.org/wiki/Q970687","display_name":"Eye tracking","level":2,"score":0.28060001134872437},{"id":"https://openalex.org/C186967261","wikidata":"https://www.wikidata.org/wiki/Q5082128","display_name":"Mobile device","level":2,"score":0.2800000011920929},{"id":"https://openalex.org/C135641252","wikidata":"https://www.wikidata.org/wiki/Q738567","display_name":"Multimodal interaction","level":2,"score":0.2718999981880188},{"id":"https://openalex.org/C89505385","wikidata":"https://www.wikidata.org/wiki/Q47146","display_name":"User interface","level":2,"score":0.25589999556541443},{"id":"https://openalex.org/C107953548","wikidata":"https://www.wikidata.org/wiki/Q1172433","display_name":"Wired glove","level":3,"score":0.2554999887943268},{"id":"https://openalex.org/C385485","wikidata":"https://www.wikidata.org/wiki/Q18907383","display_name":"Mental chronometry","level":5,"score":0.2551000118255615},{"id":"https://openalex.org/C161407221","wikidata":"https://www.wikidata.org/wiki/Q4382939","display_name":"Cognitive model","level":3,"score":0.25270000100135803},{"id":"https://openalex.org/C48007421","wikidata":"https://www.wikidata.org/wiki/Q676252","display_name":"Motion capture","level":3,"score":0.25119999051094055}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3756884.3766053","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3756884.3766053","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3756884.3766053","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2025 31st ACM Symposium on Virtual Reality Software and Technology","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3756884.3766053","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3756884.3766053","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3756884.3766053","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2025 31st ACM Symposium on Virtual Reality Software and Technology","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4417025350.pdf","grobid_xml":"https://content.openalex.org/works/W4417025350.grobid-xml"},"referenced_works_count":71,"referenced_works":["https://openalex.org/W156206789","https://openalex.org/W1623348669","https://openalex.org/W2041615160","https://openalex.org/W2045714005","https://openalex.org/W2069143585","https://openalex.org/W2084653151","https://openalex.org/W2102757824","https://openalex.org/W2157289187","https://openalex.org/W2169852119","https://openalex.org/W2487770199","https://openalex.org/W2765935135","https://openalex.org/W2782362557","https://openalex.org/W2791342729","https://openalex.org/W2889489141","https://openalex.org/W2890096158","https://openalex.org/W2906219823","https://openalex.org/W2913144876","https://openalex.org/W2915250631","https://openalex.org/W2947760526","https://openalex.org/W3025606376","https://openalex.org/W3034552520","https://openalex.org/W3103482818","https://openalex.org/W3127290767","https://openalex.org/W3135027244","https://openalex.org/W3137529719","https://openalex.org/W3161128708","https://openalex.org/W3164089640","https://openalex.org/W3165208773","https://openalex.org/W3168825659","https://openalex.org/W3176376210","https://openalex.org/W3180846684","https://openalex.org/W3193803884","https://openalex.org/W3204551889","https://openalex.org/W3208090461","https://openalex.org/W4224265009","https://openalex.org/W4225004727","https://openalex.org/W4225361088","https://openalex.org/W4225423328","https://openalex.org/W4225763874","https://openalex.org/W4230750131","https://openalex.org/W4231239346","https://openalex.org/W4250455705","https://openalex.org/W4280590039","https://openalex.org/W4281292977","https://openalex.org/W4281746271","https://openalex.org/W4285030419","https://openalex.org/W4290739304","https://openalex.org/W4294068695","https://openalex.org/W4307475498","https://openalex.org/W4309505364","https://openalex.org/W4312691565","https://openalex.org/W4312805612","https://openalex.org/W4321488259","https://openalex.org/W4328029558","https://openalex.org/W4366430409","https://openalex.org/W4366547419","https://openalex.org/W4379233238","https://openalex.org/W4380574573","https://openalex.org/W4385258623","https://openalex.org/W4387319104","https://openalex.org/W4387793613","https://openalex.org/W4389314124","https://openalex.org/W4390660258","https://openalex.org/W4393081049","https://openalex.org/W4393310037","https://openalex.org/W4402351317","https://openalex.org/W4402351643","https://openalex.org/W4406254163","https://openalex.org/W4406266237","https://openalex.org/W4408358179","https://openalex.org/W4413147024"],"related_works":[],"abstract_inverted_index":{"Enhancing":[0],"user":[1,15,72,182,257,296],"experience":[2],"and":[3,20,22,64,120,124,158,170,175,184,192,209,244,263,306],"performance,":[4],"including":[5,241],"task":[6],"load":[7,24,53,81,189,191],"in":[8,44,131],"immersive":[9,92],"environments,":[10],"requires":[11],"accurate":[12],"prediction":[13,28,42],"of":[14,71,221,256],"gaze":[16,27,157,169,183,224],"point,":[17],"reaction":[18,77,186,203,259],"time,":[19,187,204,260],"mental":[21,207,261],"physical":[23,193,212,264],"uptake.":[25],"Current":[26],"approaches":[29],"focus":[30],"primarily":[31],"on":[32,56,142,271],"motion-based":[33],"information,":[34],"lacking":[35],"physiological":[36,116,245,311],"data,":[37,94],"which":[38,95,111,146,250],"leads":[39],"to":[40,66,178,233,295,303],"poor":[41],"accuracy":[43],"highly":[45],"dynamic":[46],"virtual":[47],"reality":[48],"(VR)":[49],"environments.":[50],"Traditional":[51],"cognitive":[52,188,304],"measurements":[54],"rely":[55],"post-task":[57],"analysis":[58,232,248],"without":[59,89],"proper":[60],"multimodal":[61,107,148],"data":[62,127],"integration":[63],"fail":[65],"capture":[67],"the":[68,87,143,235,254,272],"real-time":[69,91,129],"dynamics":[70],"states":[73],"during":[74,128],"interaction.":[75],"Likewise,":[76],"time":[78],"or":[79],"attention":[80,297],"are":[82],"often":[83],"assessed":[84],"only":[85,270],"after":[86],"interaction,":[88],"using":[90],"sensor":[93],"limits":[96],"adaptive":[97],"responsiveness.":[98],"To":[99],"tackle":[100],"these":[101],"limitations,":[102],"we":[103],"leveraged":[104],"a":[105,132,137,215],"comprehensive":[106],"dataset":[108],"-":[109],"VRWalking,":[110],"recorded":[112],"timestamped":[113],"eye-tracking":[114],"metrics,":[115],"signals":[117,150],"(heart":[118],"rate":[119],"galvanic":[121],"skin":[122],"response),":[123],"behavioral":[125],"performance":[126,277,307],"engagement":[130],"VR":[133,149,286],"environment.":[134],"We":[135,226],"developed":[136],"unified":[138],"multitask":[139],"model":[140,162],"based":[141,269],"MultiPatchFormer":[144],"architecture,":[145],"processes":[147],"through":[151],"dual":[152],"patch":[153,165],"projection":[154],"branches":[155],"for":[156,202,206,211,223,284],"classification":[159,171],"inputs.":[160],"The":[161,247],"employs":[163],"multiscale":[164],"embeddings,":[166],"cross-attention":[167],"between":[168],"pathways,":[172],"channel":[173],"attention,":[174],"transformer":[176],"encoders":[177],"jointly":[179],"predict":[180],"continuous":[181],"classify":[185],"(mental":[190],"load).":[194],"Our":[195,266],"methodology":[196],"achieved":[197],"excellent":[198],"predictive":[199],"performance:":[200],"95.64%":[201],"98.01%":[205],"load,":[208,213,262,305],"97.45%":[210],"with":[214],"MAPE":[216],"(Mean":[217],"Absolute":[218],"Percentage":[219],"Error)":[220],"15.24%":[222],"prediction.":[225],"applied":[227],"Shapley":[228],"Additive":[229],"explanations":[230],"(SHAP)":[231],"interpret":[234],"model\u2019s":[236],"behavior":[237],"across":[238,278],"all":[239,279],"features,":[240],"eye-tracking,":[242],"head-tracking,":[243],"signals.":[246],"revealed":[249],"features":[251],"most":[252],"influenced":[253],"predictions":[255],"gaze,":[258],"load.":[265],"methods,":[267],"while":[268],"VRWalking":[273],"dataset,":[274],"demonstrated":[275],"strong":[276],"tasks,":[280],"suggesting":[281],"promising":[282],"potential":[283],"real-world":[285],"applications":[287],"such":[288],"as":[289],"interactive":[290],"training":[291],"systems":[292],"that":[293,301,309],"respond":[294],"lapses,":[298],"educational":[299],"platforms":[300],"adapt":[302],"assessments":[308],"consider":[310],"indicators.":[312]},"counts_by_year":[],"updated_date":"2026-03-27T05:58:40.876381","created_date":"2025-12-05T00:00:00"}