{"id":"https://openalex.org/W4402580399","doi":"https://doi.org/10.1109/tvcg.2024.3456198","title":"Gesture2Text: A Generalizable Decoder for Word-Gesture Keyboards in XR Through Trajectory Coarse Discretization and Pre-training","display_name":"Gesture2Text: A Generalizable Decoder for Word-Gesture Keyboards in XR Through Trajectory Coarse Discretization and Pre-training","publication_year":2024,"publication_date":"2024-01-01","ids":{"openalex":"https://openalex.org/W4402580399","doi":"https://doi.org/10.1109/tvcg.2024.3456198","pmid":"https://pubmed.ncbi.nlm.nih.gov/39283795"},"language":"en","primary_location":{"id":"doi:10.1109/tvcg.2024.3456198","is_oa":false,"landing_page_url":"http://dx.doi.org/10.1109/tvcg.2024.3456198","pdf_url":null,"source":{"id":"https://openalex.org/S84775595","display_name":"IEEE Transactions on Visualization and Computer Graphics","issn_l":"1077-2626","issn":["1077-2626","1941-0506","2160-9306"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Visualization and Computer Graphics","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100399628","display_name":"Junxiao Shen","orcid":"https://orcid.org/0000-0002-1552-4689"},"institutions":[{"id":"https://openalex.org/I36234482","display_name":"University of Bristol","ror":"https://ror.org/0524sp257","country_code":"GB","type":"education","lineage":["https://openalex.org/I36234482"]},{"id":"https://openalex.org/I4210128585","display_name":"META Health","ror":"https://ror.org/035h67p10","country_code":"US","type":"other","lineage":["https://openalex.org/I4210128585"]}],"countries":["GB","US"],"is_corresponding":false,"raw_author_name":"Junxiao Shen","raw_affiliation_strings":["Reality Labs Research, Meta, USA","University of Bristol, England"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Reality Labs Research, Meta, USA","institution_ids":["https://openalex.org/I4210128585"]},{"raw_affiliation_string":"University of Bristol, England","institution_ids":["https://openalex.org/I36234482"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5087975779","display_name":"Khadija Khaldi","orcid":null},"institutions":[{"id":"https://openalex.org/I4210128585","display_name":"META Health","ror":"https://ror.org/035h67p10","country_code":"US","type":"other","lineage":["https://openalex.org/I4210128585"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Khadija Khaldi","raw_affiliation_strings":["Reality Labs Research, Meta, USA","Reality Labs Research, Meta, US"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Reality Labs Research, Meta, USA","institution_ids":["https://openalex.org/I4210128585"]},{"raw_affiliation_string":"Reality Labs Research, Meta, US","institution_ids":["https://openalex.org/I4210128585"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101138388","display_name":"Enmin Zhou","orcid":null},"institutions":[{"id":"https://openalex.org/I4210128585","display_name":"META Health","ror":"https://ror.org/035h67p10","country_code":"US","type":"other","lineage":["https://openalex.org/I4210128585"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Enmin Zhou","raw_affiliation_strings":["Reality Labs Research, Meta, USA","Reality Labs Research, Meta, US"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Reality Labs Research, Meta, USA","institution_ids":["https://openalex.org/I4210128585"]},{"raw_affiliation_string":"Reality Labs Research, Meta, US","institution_ids":["https://openalex.org/I4210128585"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5072385214","display_name":"Hemant Bhaskar Surale","orcid":"https://orcid.org/0000-0001-6616-2895"},"institutions":[{"id":"https://openalex.org/I4210128585","display_name":"META Health","ror":"https://ror.org/035h67p10","country_code":"US","type":"other","lineage":["https://openalex.org/I4210128585"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Hemant Bhaskar Surale","raw_affiliation_strings":["Reality Labs Research, Meta, USA","Reality Labs Research, Meta, US"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Reality Labs Research, Meta, USA","institution_ids":["https://openalex.org/I4210128585"]},{"raw_affiliation_string":"Reality Labs Research, Meta, US","institution_ids":["https://openalex.org/I4210128585"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5034989458","display_name":"Amy Karlson","orcid":"https://orcid.org/0000-0001-8934-7761"},"institutions":[{"id":"https://openalex.org/I4210128585","display_name":"META Health","ror":"https://ror.org/035h67p10","country_code":"US","type":"other","lineage":["https://openalex.org/I4210128585"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Amy Karlson","raw_affiliation_strings":["Reality Labs Research, Meta, USA","Reality Labs Research, Meta, US"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Reality Labs Research, Meta, USA","institution_ids":["https://openalex.org/I4210128585"]},{"raw_affiliation_string":"Reality Labs Research, Meta, US","institution_ids":["https://openalex.org/I4210128585"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":5,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.140504,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"30","issue":"11","first_page":"1","last_page":"11"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10789","display_name":"Interactive and Immersive Displays","score":0.9991000294685364,"subfield":{"id":"https://openalex.org/subfields/1709","display_name":"Human-Computer Interaction"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10789","display_name":"Interactive and Immersive Displays","score":0.9991000294685364,"subfield":{"id":"https://openalex.org/subfields/1709","display_name":"Human-Computer Interaction"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10914","display_name":"Tactile and Sensory Interactions","score":0.9984999895095825,"subfield":{"id":"https://openalex.org/subfields/2805","display_name":"Cognitive Neuroscience"},"field":{"id":"https://openalex.org/fields/28","display_name":"Neuroscience"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},{"id":"https://openalex.org/T11398","display_name":"Hand Gesture Recognition Systems","score":0.9977999925613403,"subfield":{"id":"https://openalex.org/subfields/1709","display_name":"Human-Computer Interaction"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8498588800430298},{"id":"https://openalex.org/keywords/gesture","display_name":"Gesture","score":0.7583540678024292},{"id":"https://openalex.org/keywords/decoding-methods","display_name":"Decoding methods","score":0.7495366334915161},{"id":"https://openalex.org/keywords/word","display_name":"Word (group theory)","score":0.6120003461837769},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.5771970152854919},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5003566741943359},{"id":"https://openalex.org/keywords/trajectory","display_name":"Trajectory","score":0.49259239435195923},{"id":"https://openalex.org/keywords/quantization","display_name":"Quantization (signal processing)","score":0.4200618863105774},{"id":"https://openalex.org/keywords/virtual-reality","display_name":"Virtual reality","score":0.41245388984680176},{"id":"https://openalex.org/keywords/computer-vision","display_name":"Computer vision","score":0.3899945318698883},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.15155449509620667}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8498588800430298},{"id":"https://openalex.org/C207347870","wikidata":"https://www.wikidata.org/wiki/Q371174","display_name":"Gesture","level":2,"score":0.7583540678024292},{"id":"https://openalex.org/C57273362","wikidata":"https://www.wikidata.org/wiki/Q576722","display_name":"Decoding methods","level":2,"score":0.7495366334915161},{"id":"https://openalex.org/C90805587","wikidata":"https://www.wikidata.org/wiki/Q10944557","display_name":"Word (group theory)","level":2,"score":0.6120003461837769},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.5771970152854919},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5003566741943359},{"id":"https://openalex.org/C13662910","wikidata":"https://www.wikidata.org/wiki/Q193139","display_name":"Trajectory","level":2,"score":0.49259239435195923},{"id":"https://openalex.org/C28855332","wikidata":"https://www.wikidata.org/wiki/Q198099","display_name":"Quantization (signal processing)","level":2,"score":0.4200618863105774},{"id":"https://openalex.org/C194969405","wikidata":"https://www.wikidata.org/wiki/Q170519","display_name":"Virtual reality","level":2,"score":0.41245388984680176},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.3899945318698883},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.15155449509620667},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.0},{"id":"https://openalex.org/C1276947","wikidata":"https://www.wikidata.org/wiki/Q333","display_name":"Astronomy","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/tvcg.2024.3456198","is_oa":false,"landing_page_url":"http://dx.doi.org/10.1109/tvcg.2024.3456198","pdf_url":null,"source":{"id":"https://openalex.org/S84775595","display_name":"IEEE Transactions on Visualization and Computer Graphics","issn_l":"1077-2626","issn":["1077-2626","1941-0506","2160-9306"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Visualization and Computer Graphics","raw_type":"journal-article"},{"id":"pmid:39283795","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/39283795","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE transactions on visualization and computer graphics","raw_type":null}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":54,"referenced_works":["https://openalex.org/W108866686","https://openalex.org/W1495886451","https://openalex.org/W1522301498","https://openalex.org/W1561171833","https://openalex.org/W1603920809","https://openalex.org/W1984520644","https://openalex.org/W2010376411","https://openalex.org/W2012241589","https://openalex.org/W2026112871","https://openalex.org/W2064675550","https://openalex.org/W2084907998","https://openalex.org/W2099287431","https://openalex.org/W2107655853","https://openalex.org/W2125904902","https://openalex.org/W2128267659","https://openalex.org/W2133495565","https://openalex.org/W2134512786","https://openalex.org/W2402268235","https://openalex.org/W2402790201","https://openalex.org/W2611904979","https://openalex.org/W2792643794","https://openalex.org/W2795625669","https://openalex.org/W2903936639","https://openalex.org/W2923014074","https://openalex.org/W2933138175","https://openalex.org/W2941413112","https://openalex.org/W2943475692","https://openalex.org/W2955054828","https://openalex.org/W2960243907","https://openalex.org/W2979826702","https://openalex.org/W2980188738","https://openalex.org/W2980899416","https://openalex.org/W2986190770","https://openalex.org/W2998816641","https://openalex.org/W3102516861","https://openalex.org/W3118105922","https://openalex.org/W3130271024","https://openalex.org/W3163880120","https://openalex.org/W3164359705","https://openalex.org/W3176587194","https://openalex.org/W3208720208","https://openalex.org/W4224247496","https://openalex.org/W4281557260","https://openalex.org/W4288325606","https://openalex.org/W4292779060","https://openalex.org/W4295312788","https://openalex.org/W4312439180","https://openalex.org/W4315631036","https://openalex.org/W4321488104","https://openalex.org/W4360991061","https://openalex.org/W4385245566","https://openalex.org/W4387250104","https://openalex.org/W4387350466","https://openalex.org/W4387835528"],"related_works":["https://openalex.org/W2066003895","https://openalex.org/W2537963312","https://openalex.org/W2537762514","https://openalex.org/W2349788282","https://openalex.org/W577271088","https://openalex.org/W2120801881","https://openalex.org/W1982853263","https://openalex.org/W1974473538","https://openalex.org/W2020010749","https://openalex.org/W2796116650"],"abstract_inverted_index":{"Text":[0],"entry":[1],"with":[2,130,190],"word-gesture":[3,37,91,145],"keyboards":[4],"(WGK)":[5],"is":[6,155,172,208],"emerging":[7],"as":[8,54],"a":[9,14,122,134,150,175,191],"popular":[10],"method":[11],"and":[12,29,70,111,159,167,194,217],"becoming":[13],"key":[15],"interaction":[16,25],"for":[17,60,80,109,114],"Extended":[18],"Reality":[19],"(XR).":[20],"However,":[21,72],"the":[22,196,203],"diversity":[23],"of":[24,128,180],"modes,":[26],"keyboard":[27],"sizes,":[28],"visual":[30],"feedback":[31],"in":[32,45,163,221,224],"these":[33,61,73,118],"environments":[34],"introduces":[35],"divergent":[36],"trajectory":[38,92],"data":[39,93,108],"patterns,":[40],"thus":[41],"leading":[42],"to":[43,68,77,97],"complexity":[44],"decoding":[46,51,78,132],"trajectories":[47],"into":[48],"text.":[49],"Template-matching":[50],"methods,":[52],"such":[53],"SHARK2":[55,189],"[32],":[56],"are":[57,66,75],"commonly":[58],"used":[59],"WGK":[62,152,161],"systems":[63,162],"because":[64],"they":[65,100,105],"easy":[67],"implement":[69],"configure.":[71],"methods":[74],"susceptible":[76],"inaccuracies":[79],"noisy":[81],"trajectories.":[82,146],"While":[83],"conventional":[84,197],"neural-network-based":[85],"decoders":[86],"(neural":[87],"decoders)":[88],"trained":[89],"on":[90,141,182,228],"have":[94,101],"been":[95],"proposed":[96],"improve":[98],"accuracy,":[99,216],"their":[102],"own":[103],"limitations:":[104],"require":[106],"extensive":[107],"training":[110],"deep-learning":[112],"expertise":[113],"implementation.":[115],"To":[116],"address":[117],"challenges,":[119],"we":[120],"propose":[121],"novel":[123],"solution":[124],"that":[125,154],"combines":[126],"ease":[127],"implementation":[129],"high":[131],"accuracy:":[133],"generalizable":[135,156],"neural":[136,198],"decoder":[137,153,199],"enabled":[138],"by":[139,174,200],"pre-training":[140],"large-scale":[142],"coarsely":[143],"discretized":[144],"This":[147],"approach":[148],"produces":[149],"ready-to-use":[151],"across":[157],"mid-air":[158],"on-surface":[160],"augmented":[164],"reality":[165,169],"(AR)":[166],"virtual":[168],"(VR),":[170],"which":[171],"evident":[173],"robust":[176],"average":[177],"Top-4":[178],"accuracy":[179],"90.4%":[181],"four":[183],"diverse":[184],"datasets.":[185],"It":[186],"significantly":[187],"outperforms":[188],"37.2%":[192],"enhancement":[193],"surpasses":[195],"7.4%.":[201],"Moreover,":[202],"Pre-trained":[204],"Neural":[205],"Decoder's":[206],"size":[207],"only":[209],"4":[210],"MB":[211],"after":[212],"quantization,":[213],"without":[214],"sacrificing":[215],"it":[218],"can":[219],"operate":[220],"real-time,":[222],"executing":[223],"just":[225],"97":[226],"milliseconds":[227],"Quest":[229],"3.":[230]},"counts_by_year":[],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-10-10T00:00:00"}