{"id":"https://openalex.org/W4409748984","doi":"https://doi.org/10.1145/3706598.3713376","title":"Tap&Say: Touch Location-Informed Large Language Model for Multimodal Text Correction on Smartphones","display_name":"Tap&Say: Touch Location-Informed Large Language Model for Multimodal Text Correction on Smartphones","publication_year":2025,"publication_date":"2025-04-24","ids":{"openalex":"https://openalex.org/W4409748984","doi":"https://doi.org/10.1145/3706598.3713376","pmid":"https://pubmed.ncbi.nlm.nih.gov/41446598"},"language":"en","primary_location":{"id":"doi:10.1145/3706598.3713376","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3706598.3713376","pdf_url":null,"source":null,"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2025 CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3706598.3713376","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5016908947","display_name":"Maozheng Zhao","orcid":"https://orcid.org/0000-0002-2628-2523"},"institutions":[{"id":"https://openalex.org/I59553526","display_name":"Stony Brook University","ror":"https://ror.org/05qghxh33","country_code":"US","type":"education","lineage":["https://openalex.org/I59553526"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Maozheng Zhao","raw_affiliation_strings":["Department of Computer Science, Stony Brook University, Stony Brook, New York, USA"],"raw_orcid":"https://orcid.org/0000-0002-2628-2523","affiliations":[{"raw_affiliation_string":"Department of Computer Science, Stony Brook University, Stony Brook, New York, USA","institution_ids":["https://openalex.org/I59553526"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5059880425","display_name":"Michael Xuelin Huang","orcid":"https://orcid.org/0000-0001-5695-2869"},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]},{"id":"https://openalex.org/I86519309","display_name":"The University of Texas at Austin","ror":"https://ror.org/00hj54h04","country_code":"US","type":"education","lineage":["https://openalex.org/I86519309"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Michael Xuelin Huang","raw_affiliation_strings":["Google, Mountain View, California, USA","University of Texas at Austin, Austin, Texas, USA"],"raw_orcid":"https://orcid.org/0000-0001-5695-2869","affiliations":[{"raw_affiliation_string":"Google, Mountain View, California, USA","institution_ids":["https://openalex.org/I1291425158"]},{"raw_affiliation_string":"University of Texas at Austin, Austin, Texas, USA","institution_ids":["https://openalex.org/I86519309"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Nathan G Huang","orcid":"https://orcid.org/0009-0004-1676-968X"},"institutions":[{"id":"https://openalex.org/I2802960445","display_name":"IDEA Public Schools","ror":"https://ror.org/02349x007","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I2802960445"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Nathan G Huang","raw_affiliation_strings":["Westlake High School, Austin, Texas, USA"],"raw_orcid":"https://orcid.org/0009-0004-1676-968X","affiliations":[{"raw_affiliation_string":"Westlake High School, Austin, Texas, USA","institution_ids":["https://openalex.org/I2802960445"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5069085463","display_name":"Shanqing Cai","orcid":"https://orcid.org/0000-0003-4514-1715"},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Shanqing Cai","raw_affiliation_strings":["Google, Mountain View, California, USA"],"raw_orcid":"https://orcid.org/0000-0003-4514-1715","affiliations":[{"raw_affiliation_string":"Google, Mountain View, California, USA","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5026978519","display_name":"Henry Huang","orcid":null},"institutions":[{"id":"https://openalex.org/I2801851002","display_name":"Harvard University Press","ror":"https://ror.org/006v7bf86","country_code":"US","type":"other","lineage":["https://openalex.org/I136199984","https://openalex.org/I2801851002"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Henry Huang","raw_affiliation_strings":["Harvard University, Cambridge, Massachusetts, USA"],"raw_orcid":"https://orcid.org/0009-0008-9231-7633","affiliations":[{"raw_affiliation_string":"Harvard University, Cambridge, Massachusetts, USA","institution_ids":["https://openalex.org/I2801851002"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5041868265","display_name":"Michael Huang","orcid":null},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]},{"id":"https://openalex.org/I86519309","display_name":"The University of Texas at Austin","ror":"https://ror.org/00hj54h04","country_code":"US","type":"education","lineage":["https://openalex.org/I86519309"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Michael G Huang","raw_affiliation_strings":["Google, Mountain View, California, USA","University of Texas at Austin, Austin, Texas, USA"],"raw_orcid":"https://orcid.org/0009-0000-6550-163X","affiliations":[{"raw_affiliation_string":"Google, Mountain View, California, USA","institution_ids":["https://openalex.org/I1291425158"]},{"raw_affiliation_string":"University of Texas at Austin, Austin, Texas, USA","institution_ids":["https://openalex.org/I86519309"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5005293560","display_name":"Shumin Zhai","orcid":"https://orcid.org/0000-0003-0752-2090"},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Shumin Zhai","raw_affiliation_strings":["Google, Mountain View, California, USA"],"raw_orcid":"https://orcid.org/0000-0003-0752-2090","affiliations":[{"raw_affiliation_string":"Google, Mountain View, California, USA","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5005648942","display_name":"I. V. Ramakrishnan","orcid":"https://orcid.org/0000-0002-1768-7043"},"institutions":[{"id":"https://openalex.org/I59553526","display_name":"Stony Brook University","ror":"https://ror.org/05qghxh33","country_code":"US","type":"education","lineage":["https://openalex.org/I59553526"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"IV Ramakrishnan","raw_affiliation_strings":["Department of Computer Science, Stony Brook University, Stony Brook, New York, USA"],"raw_orcid":"https://orcid.org/0000-0002-1768-7043","affiliations":[{"raw_affiliation_string":"Department of Computer Science, Stony Brook University, Stony Brook, New York, USA","institution_ids":["https://openalex.org/I59553526"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5016901770","display_name":"Xiaojun Bi","orcid":"https://orcid.org/0000-0002-9716-7709"},"institutions":[{"id":"https://openalex.org/I59553526","display_name":"Stony Brook University","ror":"https://ror.org/05qghxh33","country_code":"US","type":"education","lineage":["https://openalex.org/I59553526"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Xiaojun Bi","raw_affiliation_strings":["Department of Computer Science, Stony Brook University, Stony Brook, New York, USA"],"raw_orcid":"https://orcid.org/0000-0002-9716-7709","affiliations":[{"raw_affiliation_string":"Department of Computer Science, Stony Brook University, Stony Brook, New York, USA","institution_ids":["https://openalex.org/I59553526"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":9,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":8.4395,"has_fulltext":false,"cited_by_count":5,"citation_normalized_percentile":{"value":0.97140839,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":95,"max":99},"biblio":{"volume":"2025","issue":null,"first_page":"1","last_page":"17"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11439","display_name":"Video Analysis and Summarization","score":0.9919999837875366,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12720","display_name":"Multimedia Communication and Technology","score":0.991100013256073,"subfield":{"id":"https://openalex.org/subfields/3312","display_name":"Sociology and Political Science"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6665773391723633},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.5433658957481384},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.4798581004142761},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.3738235831260681},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3545961081981659},{"id":"https://openalex.org/keywords/multimedia","display_name":"Multimedia","score":0.34530705213546753},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.3415634334087372},{"id":"https://openalex.org/keywords/computer-vision","display_name":"Computer vision","score":0.34012508392333984}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6665773391723633},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.5433658957481384},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.4798581004142761},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3738235831260681},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3545961081981659},{"id":"https://openalex.org/C49774154","wikidata":"https://www.wikidata.org/wiki/Q131765","display_name":"Multimedia","level":1,"score":0.34530705213546753},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.3415634334087372},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.34012508392333984}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1145/3706598.3713376","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3706598.3713376","pdf_url":null,"source":null,"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2025 CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"},{"id":"pmid:41446598","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/41446598","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the SIGCHI conference on human factors in computing systems. CHI Conference","raw_type":null},{"id":"pmh:oai:pubmedcentral.nih.gov:12723524","is_oa":true,"landing_page_url":"https://pmc.ncbi.nlm.nih.gov/articles/PMC12723524/","pdf_url":null,"source":{"id":"https://openalex.org/S2764455111","display_name":"PubMed Central","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Proc SIGCHI Conf Hum Factor Comput Syst","raw_type":"Text"}],"best_oa_location":{"id":"doi:10.1145/3706598.3713376","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3706598.3713376","pdf_url":null,"source":null,"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2025 CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education","score":0.699999988079071}],"awards":[{"id":"https://openalex.org/G3335282361","display_name":null,"funder_award_id":"2113485","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G5924404795","display_name":null,"funder_award_id":"R01EY03568801,R01EY03008503","funder_id":"https://openalex.org/F4320332161","funder_display_name":"National Institutes of Health"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320332161","display_name":"National Institutes of Health","ror":"https://ror.org/01cwqze88"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":26,"referenced_works":["https://openalex.org/W1993223663","https://openalex.org/W2041977675","https://openalex.org/W2052404777","https://openalex.org/W2058049219","https://openalex.org/W2069657084","https://openalex.org/W2077940530","https://openalex.org/W2098924240","https://openalex.org/W2108518773","https://openalex.org/W2119101732","https://openalex.org/W2124144842","https://openalex.org/W2124810338","https://openalex.org/W2280355185","https://openalex.org/W2295007882","https://openalex.org/W2399327359","https://openalex.org/W2597684388","https://openalex.org/W2942185243","https://openalex.org/W2942986209","https://openalex.org/W2979826702","https://openalex.org/W2980850520","https://openalex.org/W3093820610","https://openalex.org/W3206276539","https://openalex.org/W3207043055","https://openalex.org/W4220776662","https://openalex.org/W4385569962","https://openalex.org/W4389519306","https://openalex.org/W4396833729"],"related_works":["https://openalex.org/W2169518243","https://openalex.org/W304855073","https://openalex.org/W938111471","https://openalex.org/W2252095989","https://openalex.org/W2511882081","https://openalex.org/W2518250394","https://openalex.org/W2106335228","https://openalex.org/W2032169142","https://openalex.org/W2105076537","https://openalex.org/W2267221041"],"abstract_inverted_index":{"layer":[0],"that":[1,50],"integrates":[2],"the":[3,7,15,23,40],"tap":[4,16],"location":[5,17],"into":[6],"LLM's":[8],"attention":[9],"mechanism,":[10],"enabling":[11],"it":[12],"to":[13],"utilize":[14],"for":[18],"text":[19],"correction.":[20],"We":[21],"fine-tuned":[22],"touch":[24,29],"location-informed":[25],"LLM":[26],"on":[27],"synthetic":[28],"locations":[30],"and":[31,61,66],"correction":[32,37],"commands,":[33],"achieving":[34],"significantly":[35],"higher":[36],"accuracy":[38],"than":[39],"state-of-the-art":[41],"method":[42],"VT":[43,53],"[45].":[44],"A":[45],"16-person":[46],"user":[47],"study":[48],"demonstrated":[49],"Tap&Say":[51],"outperforms":[52],"[45]":[54],"with":[55],"16.4%":[56],"shorter":[57],"task":[58],"completion":[59],"time":[60],"47.5%":[62],"fewer":[63],"keyboard":[64],"clicks":[65],"is":[67],"preferred":[68],"by":[69],"users.":[70]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":2}],"updated_date":"2026-06-19T17:40:00.097472","created_date":"2025-10-10T00:00:00"}