{"id":"https://openalex.org/W4396832321","doi":"https://doi.org/10.1145/3613904.3642068","title":"OmniActions: Predicting Digital Actions in Response to Real-World Multimodal Sensory Inputs with LLMs","display_name":"OmniActions: Predicting Digital Actions in Response to Real-World Multimodal Sensory Inputs with LLMs","publication_year":2024,"publication_date":"2024-05-11","ids":{"openalex":"https://openalex.org/W4396832321","doi":"https://doi.org/10.1145/3613904.3642068"},"language":"en","primary_location":{"id":"doi:10.1145/3613904.3642068","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3613904.3642068","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3613904.3642068","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3613904.3642068","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5028570020","display_name":"Jiahao Nick Li","orcid":"https://orcid.org/0000-0002-4937-0024"},"institutions":[{"id":"https://openalex.org/I161318765","display_name":"University of California, Los Angeles","ror":"https://ror.org/046rm7j60","country_code":"US","type":"education","lineage":["https://openalex.org/I161318765"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Jiahao Nick Li","raw_affiliation_strings":["HCI Research, University of California, Los Angeles, United States"],"raw_orcid":"https://orcid.org/0000-0002-4937-0024","affiliations":[{"raw_affiliation_string":"HCI Research, University of California, Los Angeles, United States","institution_ids":["https://openalex.org/I161318765"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5064730192","display_name":"Yan Xu","orcid":"https://orcid.org/0009-0005-3890-9847"},"institutions":[{"id":"https://openalex.org/I4210128585","display_name":"META Health","ror":"https://ror.org/035h67p10","country_code":"US","type":"other","lineage":["https://openalex.org/I4210128585"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yan Xu","raw_affiliation_strings":["Reality Labs Research, Meta, United States"],"raw_orcid":"https://orcid.org/0009-0005-3890-9847","affiliations":[{"raw_affiliation_string":"Reality Labs Research, Meta, United States","institution_ids":["https://openalex.org/I4210128585"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5058485609","display_name":"Tovi Grossman","orcid":"https://orcid.org/0000-0002-0494-5373"},"institutions":[{"id":"https://openalex.org/I185261750","display_name":"University of Toronto","ror":"https://ror.org/03dbr7087","country_code":"CA","type":"education","lineage":["https://openalex.org/I185261750"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Tovi Grossman","raw_affiliation_strings":["Department of Computer Science, University of Toronto, Canada"],"raw_orcid":"https://orcid.org/0000-0002-0494-5373","affiliations":[{"raw_affiliation_string":"Department of Computer Science, University of Toronto, Canada","institution_ids":["https://openalex.org/I185261750"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5000507697","display_name":"Stephanie Santosa","orcid":"https://orcid.org/0000-0002-6010-005X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Stephanie Santosa","raw_affiliation_strings":["Reality Labs Research, Meta, Canada"],"raw_orcid":"https://orcid.org/0000-0002-6010-005X","affiliations":[{"raw_affiliation_string":"Reality Labs Research, Meta, Canada","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5075344578","display_name":"M. Li","orcid":"https://orcid.org/0000-0002-0558-7643"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Michelle Li","raw_affiliation_strings":["Reality Labs Research, Meta, Canada"],"raw_orcid":"https://orcid.org/0000-0002-0558-7643","affiliations":[{"raw_affiliation_string":"Reality Labs Research, Meta, Canada","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5028570020"],"corresponding_institution_ids":["https://openalex.org/I161318765"],"apc_list":null,"apc_paid":null,"fwci":7.4853,"has_fulltext":false,"cited_by_count":23,"citation_normalized_percentile":{"value":0.97965961,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"22"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10914","display_name":"Tactile and Sensory Interactions","score":0.9968000054359436,"subfield":{"id":"https://openalex.org/subfields/2805","display_name":"Cognitive Neuroscience"},"field":{"id":"https://openalex.org/fields/28","display_name":"Neuroscience"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},"topics":[{"id":"https://openalex.org/T10914","display_name":"Tactile and Sensory Interactions","score":0.9968000054359436,"subfield":{"id":"https://openalex.org/subfields/2805","display_name":"Cognitive Neuroscience"},"field":{"id":"https://openalex.org/fields/28","display_name":"Neuroscience"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},{"id":"https://openalex.org/T12128","display_name":"AI in Service Interactions","score":0.9944999814033508,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10803","display_name":"Innovative Human-Technology Interaction","score":0.9936000108718872,"subfield":{"id":"https://openalex.org/subfields/1709","display_name":"Human-Computer Interaction"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7809616327285767},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.7224925756454468},{"id":"https://openalex.org/keywords/pipeline","display_name":"Pipeline (software)","score":0.6899377703666687},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.6580905318260193},{"id":"https://openalex.org/keywords/action","display_name":"Action (physics)","score":0.570923924446106},{"id":"https://openalex.org/keywords/multimodal-interaction","display_name":"Multimodal interaction","score":0.5650575757026672},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.5423044562339783},{"id":"https://openalex.org/keywords/multimodality","display_name":"Multimodality","score":0.4913307726383209},{"id":"https://openalex.org/keywords/interaction-design","display_name":"Interaction design","score":0.41527003049850464},{"id":"https://openalex.org/keywords/multimedia","display_name":"Multimedia","score":0.33252036571502686},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.17944249510765076},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.09931626915931702}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7809616327285767},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.7224925756454468},{"id":"https://openalex.org/C43521106","wikidata":"https://www.wikidata.org/wiki/Q2165493","display_name":"Pipeline (software)","level":2,"score":0.6899377703666687},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.6580905318260193},{"id":"https://openalex.org/C2780791683","wikidata":"https://www.wikidata.org/wiki/Q846785","display_name":"Action (physics)","level":2,"score":0.570923924446106},{"id":"https://openalex.org/C135641252","wikidata":"https://www.wikidata.org/wiki/Q738567","display_name":"Multimodal interaction","level":2,"score":0.5650575757026672},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.5423044562339783},{"id":"https://openalex.org/C2780910867","wikidata":"https://www.wikidata.org/wiki/Q1952416","display_name":"Multimodality","level":2,"score":0.4913307726383209},{"id":"https://openalex.org/C13854087","wikidata":"https://www.wikidata.org/wiki/Q1061656","display_name":"Interaction design","level":2,"score":0.41527003049850464},{"id":"https://openalex.org/C49774154","wikidata":"https://www.wikidata.org/wiki/Q131765","display_name":"Multimedia","level":1,"score":0.33252036571502686},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.17944249510765076},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.09931626915931702},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3613904.3642068","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3613904.3642068","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3613904.3642068","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3613904.3642068","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3613904.3642068","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3613904.3642068","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","score":0.5600000023841858,"display_name":"Quality Education"}],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":false},"content_urls":{"pdf":"https://content.openalex.org/works/W4396832321.pdf"},"referenced_works_count":39,"referenced_works":["https://openalex.org/W1499226393","https://openalex.org/W1536680647","https://openalex.org/W1551137640","https://openalex.org/W1966642888","https://openalex.org/W1992221677","https://openalex.org/W1994349244","https://openalex.org/W1995435389","https://openalex.org/W1995635152","https://openalex.org/W2024486844","https://openalex.org/W2070671805","https://openalex.org/W2096091650","https://openalex.org/W2110277250","https://openalex.org/W2126145938","https://openalex.org/W2141990085","https://openalex.org/W2170723298","https://openalex.org/W2172064665","https://openalex.org/W2285672838","https://openalex.org/W2311815372","https://openalex.org/W2405622356","https://openalex.org/W2739069594","https://openalex.org/W2768343520","https://openalex.org/W2895817099","https://openalex.org/W2963712589","https://openalex.org/W2978533520","https://openalex.org/W2980956911","https://openalex.org/W3003645575","https://openalex.org/W3005279352","https://openalex.org/W3021091026","https://openalex.org/W3106143313","https://openalex.org/W3152835281","https://openalex.org/W3205786327","https://openalex.org/W4221047484","https://openalex.org/W4221143046","https://openalex.org/W4307475457","https://openalex.org/W4319167005","https://openalex.org/W4366547453","https://openalex.org/W4366729084","https://openalex.org/W6665580293","https://openalex.org/W6778883912"],"related_works":["https://openalex.org/W2013013717","https://openalex.org/W66256988","https://openalex.org/W2068964259","https://openalex.org/W4302857920","https://openalex.org/W1987340722","https://openalex.org/W199123384","https://openalex.org/W2739856905","https://openalex.org/W4239305747","https://openalex.org/W4205486898","https://openalex.org/W2166331011"],"abstract_inverted_index":{"The":[0],"progression":[1],"to":[2,9,30,54,77,86,122,217],"\u201cPervasive":[3],"Augmented":[4],"Reality\u201d":[5],"envisions":[6],"easy":[7],"access":[8,53],"multimodal":[10,34,126,143],"information":[11,35,153],"continuously.":[12],"However,":[13],"in":[14,39,120,155,165],"many":[15],"everyday":[16],"scenarios,":[17],"users":[18,37],"are":[19],"occupied":[20],"physically,":[21],"cognitively":[22],"or":[23,92],"socially.":[24],"This":[25],"may":[26],"increase":[27],"the":[28,33,40,63,81,151,156,161,166,187,199,218],"friction":[29],"act":[31],"upon":[32],"that":[36,74,83,116,141],"encounter":[38],"world.":[41],"To":[42,61],"reduce":[43],"such":[44],"friction,":[45],"future":[46],"interactive":[47,204],"interfaces":[48],"should":[49],"intelligently":[50],"provide":[51],"quick":[52],"digital":[55,67,113],"actions":[56,88,98,115,149],"based":[57],"on":[58,89,150,173],"users\u2019":[59],"context.":[60],"explore":[62],"range":[64],"of":[65,112,125,176,198],"possible":[66],"actions,":[68],"we":[69,106,169,201],"conducted":[70],"a":[71,108,133],"diary":[72,167],"study":[73],"required":[75],"participants":[76],"capture":[78],"and":[79,99,146,183,185,206,215,221],"share":[80],"media":[82],"they":[84],"intended":[85],"perform":[87],"(e.g.,":[90],"images":[91],"audio),":[93],"along":[94],"with":[95],"their":[96],"desired":[97],"other":[100],"contextual":[101],"information.":[102],"Using":[103,160],"this":[104],"data,":[105],"generated":[107],"holistic":[109],"design":[110,158],"space":[111],"follow-up":[114,148],"could":[117],"be":[118],"performed":[119,170],"response":[121],"different":[123],"types":[124],"sensory":[127,144],"inputs.":[128],"We":[129],"then":[130],"designed":[131],"OmniActions,":[132],"pipeline":[134],"powered":[135],"by":[136],"large":[137],"language":[138],"models":[139],"(LLMs)":[140],"processes":[142],"inputs":[145],"predicts":[147],"target":[152],"grounded":[154],"derived":[157],"space.":[159],"empirical":[162],"data":[163],"collected":[164],"study,":[168],"quantitative":[171],"evaluations":[172],"three":[174],"variations":[175],"LLM":[177],"techniques":[178],"(intent":[179],"classification,":[180],"in-context":[181],"learning":[182],"finetuning)":[184],"identified":[186],"most":[188],"effective":[189],"technique":[190],"for":[191],"our":[192],"task.":[193],"Additionally,":[194],"as":[195],"an":[196,203],"instantiation":[197],"pipeline,":[200],"developed":[202],"prototype":[205],"reported":[207],"preliminary":[208],"user":[209],"feedback":[210],"about":[211],"how":[212],"people":[213],"perceive":[214],"react":[216],"action":[219],"predictions":[220],"its":[222],"errors.":[223]},"counts_by_year":[{"year":2026,"cited_by_count":4},{"year":2025,"cited_by_count":14},{"year":2024,"cited_by_count":5}],"updated_date":"2026-03-27T05:58:40.876381","created_date":"2025-10-10T00:00:00"}