{"id":"https://openalex.org/W4409882920","doi":"https://doi.org/10.1109/tits.2025.3558085","title":"Human-Factors-in-Aviation-Loop: Multimodal Deep Learning for Pilot Situation Awareness Analysis Using Gaze Position and Flight Control Data","display_name":"Human-Factors-in-Aviation-Loop: Multimodal Deep Learning for Pilot Situation Awareness Analysis Using Gaze Position and Flight Control Data","publication_year":2025,"publication_date":"2025-04-28","ids":{"openalex":"https://openalex.org/W4409882920","doi":"https://doi.org/10.1109/tits.2025.3558085"},"language":"en","primary_location":{"id":"doi:10.1109/tits.2025.3558085","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tits.2025.3558085","pdf_url":null,"source":{"id":"https://openalex.org/S144771191","display_name":"IEEE Transactions on Intelligent Transportation Systems","issn_l":"1524-9050","issn":["1524-9050","1558-0016"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Intelligent Transportation Systems","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://figshare.com/articles/journal_contribution/Human-Factors-in-Aviation-Loop_Multimodal_deep_learning_for_pilot_situation_awareness_analysis_using_gaze_position_and_flight_control_data/28890341","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5030623154","display_name":"Jiawei Xu","orcid":"https://orcid.org/0000-0001-9758-0563"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gary J. W. Xu","raw_affiliation_strings":["The Second Research Institute of CAAC, Chengdu, China"],"raw_orcid":"https://orcid.org/0000-0001-9758-0563","affiliations":[{"raw_affiliation_string":"The Second Research Institute of CAAC, Chengdu, China","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5018823373","display_name":"Sicheng Pan","orcid":"https://orcid.org/0000-0002-6705-7322"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sicheng Pan","raw_affiliation_strings":["The Second Research Institute of CAAC, Chengdu, China"],"raw_orcid":"https://orcid.org/0000-0002-6705-7322","affiliations":[{"raw_affiliation_string":"The Second Research Institute of CAAC, Chengdu, China","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5043291162","display_name":"Poly Z. H. Sun","orcid":"https://orcid.org/0000-0001-6156-1766"},"institutions":[{"id":"https://openalex.org/I66867065","display_name":"East China Normal University","ror":"https://ror.org/02n96ep67","country_code":"CN","type":"education","lineage":["https://openalex.org/I66867065"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Poly Z. H. Sun","raw_affiliation_strings":["Department of Artificial Intelligence and Human Factors Engineering, School of Psychology and Cognitive Science, East China Normal University, Shanghai, China"],"raw_orcid":"https://orcid.org/0000-0001-6156-1766","affiliations":[{"raw_affiliation_string":"Department of Artificial Intelligence and Human Factors Engineering, School of Psychology and Cognitive Science, East China Normal University, Shanghai, China","institution_ids":["https://openalex.org/I66867065"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100765129","display_name":"Kun Guo","orcid":"https://orcid.org/0000-0001-6765-1957"},"institutions":[{"id":"https://openalex.org/I51532219","display_name":"University of Lincoln","ror":"https://ror.org/03yeq9x20","country_code":"GB","type":"education","lineage":["https://openalex.org/I51532219"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Kun Guo","raw_affiliation_strings":["School of Psychology, University of Lincoln, Lincoln, U.K"],"raw_orcid":"https://orcid.org/0000-0001-6765-1957","affiliations":[{"raw_affiliation_string":"School of Psychology, University of Lincoln, Lincoln, U.K","institution_ids":["https://openalex.org/I51532219"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5007177066","display_name":"Seop Hyeong Park","orcid":"https://orcid.org/0000-0001-9570-7575"},"institutions":[{"id":"https://openalex.org/I146824383","display_name":"Hallym University","ror":"https://ror.org/03sbhge02","country_code":"KR","type":"education","lineage":["https://openalex.org/I146824383"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Seop Hyeong Park","raw_affiliation_strings":["Division of Software, Hallym University, Chuncheon, Gangwon-do, South Korea"],"raw_orcid":"https://orcid.org/0000-0001-9570-7575","affiliations":[{"raw_affiliation_string":"Division of Software, Hallym University, Chuncheon, Gangwon-do, South Korea","institution_ids":["https://openalex.org/I146824383"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5070678177","display_name":"Fengshuo Yan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Fengshuo Yan","raw_affiliation_strings":["The Second Research Institute of CAAC, Chengdu, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The Second Research Institute of CAAC, Chengdu, China","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5063869967","display_name":"Mingyang Gao","orcid":"https://orcid.org/0000-0002-8304-7909"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mingyang Gao","raw_affiliation_strings":["The Second Research Institute of CAAC, Chengdu, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The Second Research Institute of CAAC, Chengdu, China","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5001599231","display_name":"Xiaoru Wanyan","orcid":"https://orcid.org/0000-0003-4497-2242"},"institutions":[{"id":"https://openalex.org/I82880672","display_name":"Beihang University","ror":"https://ror.org/00wk2mp56","country_code":"CN","type":"education","lineage":["https://openalex.org/I82880672"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xiaoru Wanyan","raw_affiliation_strings":["School of Aeronautic Science and Engineering, Beihang University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0003-4497-2242","affiliations":[{"raw_affiliation_string":"School of Aeronautic Science and Engineering, Beihang University, Beijing, China","institution_ids":["https://openalex.org/I82880672"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5042965582","display_name":"Hong Cheng","orcid":"https://orcid.org/0000-0001-5532-9530"},"institutions":[{"id":"https://openalex.org/I150229711","display_name":"University of Electronic Science and Technology of China","ror":"https://ror.org/04qr3zq92","country_code":"CN","type":"education","lineage":["https://openalex.org/I150229711"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Hong Cheng","raw_affiliation_strings":["School of Automation Engineering, University of Electronic Science and Technology of China, Chengdu, China"],"raw_orcid":"https://orcid.org/0000-0001-5532-9530","affiliations":[{"raw_affiliation_string":"School of Automation Engineering, University of Electronic Science and Technology of China, Chengdu, China","institution_ids":["https://openalex.org/I150229711"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5110956301","display_name":"Edmond Q. Wu","orcid":"https://orcid.org/0000-0003-1301-9870"},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Edmond Q. Wu","raw_affiliation_strings":["Department of Automation, Shanghai Jiao Tong University, Shanghai, China"],"raw_orcid":"https://orcid.org/0000-0003-1301-9870","affiliations":[{"raw_affiliation_string":"Department of Automation, Shanghai Jiao Tong University, Shanghai, China","institution_ids":["https://openalex.org/I183067930"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":6,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":61.1647,"has_fulltext":false,"cited_by_count":42,"citation_normalized_percentile":{"value":0.9995562,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":99,"max":100},"biblio":{"volume":"26","issue":"6","first_page":"8065","last_page":"8077"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10525","display_name":"Human-Automation Interaction and Safety","score":0.9943000078201294,"subfield":{"id":"https://openalex.org/subfields/3207","display_name":"Social Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10525","display_name":"Human-Automation Interaction and Safety","score":0.9943000078201294,"subfield":{"id":"https://openalex.org/subfields/3207","display_name":"Social Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11325","display_name":"Inertial Sensor and Navigation","score":0.9320999979972839,"subfield":{"id":"https://openalex.org/subfields/2202","display_name":"Aerospace Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12125","display_name":"Aerospace and Aviation Technology","score":0.9078999757766724,"subfield":{"id":"https://openalex.org/subfields/2202","display_name":"Aerospace Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/gaze","display_name":"Gaze","score":0.7091517448425293},{"id":"https://openalex.org/keywords/aviation-accident","display_name":"Aviation accident","score":0.6582639217376709},{"id":"https://openalex.org/keywords/aviation","display_name":"Aviation","score":0.5648032426834106},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5540256500244141},{"id":"https://openalex.org/keywords/position","display_name":"Position (finance)","score":0.539841890335083},{"id":"https://openalex.org/keywords/human-in-the-loop","display_name":"Human-in-the-loop","score":0.5149391293525696},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.500910758972168},{"id":"https://openalex.org/keywords/aeronautics","display_name":"Aeronautics","score":0.4958086907863617},{"id":"https://openalex.org/keywords/situation-awareness","display_name":"Situation awareness","score":0.48107242584228516},{"id":"https://openalex.org/keywords/deep-learning","display_name":"Deep learning","score":0.4679882228374481},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.3955481946468353},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.3454616665840149},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.33358824253082275},{"id":"https://openalex.org/keywords/computer-vision","display_name":"Computer vision","score":0.3284478187561035},{"id":"https://openalex.org/keywords/aerospace-engineering","display_name":"Aerospace engineering","score":0.15659460425376892}],"concepts":[{"id":"https://openalex.org/C2779916870","wikidata":"https://www.wikidata.org/wiki/Q14467155","display_name":"Gaze","level":2,"score":0.7091517448425293},{"id":"https://openalex.org/C2908613842","wikidata":"https://www.wikidata.org/wiki/Q108284447","display_name":"Aviation accident","level":3,"score":0.6582639217376709},{"id":"https://openalex.org/C74448152","wikidata":"https://www.wikidata.org/wiki/Q765633","display_name":"Aviation","level":2,"score":0.5648032426834106},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5540256500244141},{"id":"https://openalex.org/C198082294","wikidata":"https://www.wikidata.org/wiki/Q3399648","display_name":"Position (finance)","level":2,"score":0.539841890335083},{"id":"https://openalex.org/C2780626000","wikidata":"https://www.wikidata.org/wiki/Q5936775","display_name":"Human-in-the-loop","level":2,"score":0.5149391293525696},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.500910758972168},{"id":"https://openalex.org/C178802073","wikidata":"https://www.wikidata.org/wiki/Q8421","display_name":"Aeronautics","level":1,"score":0.4958086907863617},{"id":"https://openalex.org/C145804949","wikidata":"https://www.wikidata.org/wiki/Q478123","display_name":"Situation awareness","level":2,"score":0.48107242584228516},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.4679882228374481},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.3955481946468353},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.3454616665840149},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.33358824253082275},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.3284478187561035},{"id":"https://openalex.org/C146978453","wikidata":"https://www.wikidata.org/wiki/Q3798668","display_name":"Aerospace engineering","level":1,"score":0.15659460425376892},{"id":"https://openalex.org/C10138342","wikidata":"https://www.wikidata.org/wiki/Q43015","display_name":"Finance","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/tits.2025.3558085","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tits.2025.3558085","pdf_url":null,"source":{"id":"https://openalex.org/S144771191","display_name":"IEEE Transactions on Intelligent Transportation Systems","issn_l":"1524-9050","issn":["1524-9050","1558-0016"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Intelligent Transportation Systems","raw_type":"journal-article"},{"id":"pmh:oai:figshare.com:article/28890341","is_oa":true,"landing_page_url":"https://figshare.com/articles/journal_contribution/Human-Factors-in-Aviation-Loop_Multimodal_deep_learning_for_pilot_situation_awareness_analysis_using_gaze_position_and_flight_control_data/28890341","pdf_url":null,"source":{"id":"https://openalex.org/S4377196282","display_name":"Figshare","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I4210132348","host_organization_name":"Figshare (United Kingdom)","host_organization_lineage":["https://openalex.org/I4210132348"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Text"}],"best_oa_location":{"id":"pmh:oai:figshare.com:article/28890341","is_oa":true,"landing_page_url":"https://figshare.com/articles/journal_contribution/Human-Factors-in-Aviation-Loop_Multimodal_deep_learning_for_pilot_situation_awareness_analysis_using_gaze_position_and_flight_control_data/28890341","pdf_url":null,"source":{"id":"https://openalex.org/S4377196282","display_name":"Figshare","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I4210132348","host_organization_name":"Figshare (United Kingdom)","host_organization_lineage":["https://openalex.org/I4210132348"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":56,"referenced_works":["https://openalex.org/W17263963","https://openalex.org/W1494957782","https://openalex.org/W1595998849","https://openalex.org/W1967723072","https://openalex.org/W1969145999","https://openalex.org/W1975538040","https://openalex.org/W2000631513","https://openalex.org/W2018988377","https://openalex.org/W2028822651","https://openalex.org/W2069757512","https://openalex.org/W2093887643","https://openalex.org/W2105029246","https://openalex.org/W2109320267","https://openalex.org/W2137669830","https://openalex.org/W2142578527","https://openalex.org/W2146948159","https://openalex.org/W2551795556","https://openalex.org/W2789244099","https://openalex.org/W2892741787","https://openalex.org/W2897997731","https://openalex.org/W2917267381","https://openalex.org/W2931511425","https://openalex.org/W2990210508","https://openalex.org/W3006585575","https://openalex.org/W3011785450","https://openalex.org/W3035123951","https://openalex.org/W3119728133","https://openalex.org/W3124293960","https://openalex.org/W3127561923","https://openalex.org/W3128725809","https://openalex.org/W3134517737","https://openalex.org/W3137585737","https://openalex.org/W3142351219","https://openalex.org/W3153269070","https://openalex.org/W3162665729","https://openalex.org/W3168997536","https://openalex.org/W3176912151","https://openalex.org/W3197375243","https://openalex.org/W3214662938","https://openalex.org/W4210300019","https://openalex.org/W4226149230","https://openalex.org/W4226396566","https://openalex.org/W4292981732","https://openalex.org/W4312242979","https://openalex.org/W4312966716","https://openalex.org/W4313219301","https://openalex.org/W4313340670","https://openalex.org/W4319459199","https://openalex.org/W4382119270","https://openalex.org/W4385194695","https://openalex.org/W4385245566","https://openalex.org/W4400018327","https://openalex.org/W6631190155","https://openalex.org/W6745609711","https://openalex.org/W6746580549","https://openalex.org/W6792519600"],"related_works":["https://openalex.org/W2937394097","https://openalex.org/W2093172766","https://openalex.org/W4307490329","https://openalex.org/W46795511","https://openalex.org/W2115815699","https://openalex.org/W590722238","https://openalex.org/W2026593924","https://openalex.org/W1583280211","https://openalex.org/W2082305874","https://openalex.org/W3193068826"],"abstract_inverted_index":{"Situation":[0,78],"awareness":[1],"(SA)":[2],"is":[3],"a":[4,32,67,107,144,216],"crucial":[5],"factor":[6],"affecting":[7],"flight":[8,54,68,93,122,169,212,225],"safety":[9,213],"for":[10,21,191,207],"pilots,":[11,22],"yet":[12],"few":[13],"studies":[14],"have":[15],"focused":[16],"specifically":[17],"on":[18,111],"modeling":[19],"SA":[20,72,136,186,192,222],"resulting":[23],"in":[24,172,194],"limited":[25],"success.":[26],"In":[27],"this":[28],"paper,":[29],"we":[30,132],"propose":[31],"novel":[33],"multimodal":[34,154,201],"deep":[35,47,97,108,127,202],"learning":[36,203],"approach":[37,43,204],"to":[38],"monitor":[39],"pilots\u2019":[40,221],"SA.":[41,174],"The":[42,84,149],"combines":[44],"handcrafted":[45,85,116],"and":[46,53,91,121,187,196,211],"features":[48,86,98,117],"obtained":[49,75],"from":[50,58,101],"eye":[51,89,119,130,162],"movement":[52,120,163],"control":[55,94,170],"data":[56],"collected":[57],"27":[59],"novice":[60],"pilots":[61],"across":[62],"different":[63],"training":[64,210],"phases":[65],"using":[66,76,106],"simulator.":[69],"Ground":[70],"truth":[71],"measurements":[73],"were":[74,99],"the":[77,115,134,153,157,161,168],"Awareness":[79],"Global":[80],"Assessment":[81],"Technique":[82],"(SAGAT).":[83],"included":[87],"13":[88],"movements":[90],"22":[92],"features,":[95],"while":[96],"extracted":[100],"time-series":[102],"of":[103,118,129,147,183,220],"gaze":[104],"positions":[105],"extractor":[109],"based":[110],"Transformer.":[112],"By":[113],"fusing":[114],"control,":[123],"along":[124],"with":[125,160],"one":[126],"feature":[128],"movement,":[131],"predicted":[133],"final":[135],"level.":[137],"Through":[138],"leave-one-flight-out":[139],"cross-validation,":[140],"our":[141,177,200],"model":[142,155],"achieved":[143],"higher":[145],"accuracy":[146],"92.04%.":[148],"results":[150],"indicate":[151],"that":[152],"outperforms":[156],"unimodal":[158],"models,":[159],"modality":[164,171],"demonstrating":[165],"superiority":[166],"over":[167],"predicting":[173,184],"This":[175],"suggests":[176],"method":[178],"provides":[179],"an":[180],"objective":[181],"means":[182],"pilot\u2019s":[185],"offers":[188],"new":[189],"insights":[190],"assessment":[193],"aviation":[195],"other":[197],"fields.":[198],"Overall,":[199],"holds":[205],"promise":[206],"enhancing":[208],"pilot":[209],"by":[214],"facilitating":[215],"more":[217],"comprehensive":[218],"understanding":[219],"during":[223],"critical":[224],"scenarios.":[226]},"counts_by_year":[{"year":2026,"cited_by_count":29},{"year":2025,"cited_by_count":13}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}