{"id":"https://openalex.org/W4410068087","doi":"https://doi.org/10.1145/3715014.3724366","title":"An XR Platform that Integrates Large Language Models with the Physical World","display_name":"An XR Platform that Integrates Large Language Models with the Physical World","publication_year":2025,"publication_date":"2025-05-04","ids":{"openalex":"https://openalex.org/W4410068087","doi":"https://doi.org/10.1145/3715014.3724366"},"language":"en","primary_location":{"id":"doi:10.1145/3715014.3724366","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3715014.3724366","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 23rd ACM Conference on Embedded Networked Sensor Systems","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5005699829","display_name":"Sruti Srinidhi","orcid":"https://orcid.org/0000-0002-4346-5305"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Sruti Srinidhi","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, PA, USA"],"raw_orcid":"https://orcid.org/0000-0002-4346-5305","affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, PA, USA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5072397325","display_name":"Edward Lu","orcid":"https://orcid.org/0009-0007-5627-0244"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Edward Lu","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, PA, USA"],"raw_orcid":"https://orcid.org/0009-0007-5627-0244","affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, PA, USA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5117427923","display_name":"Akul Singh","orcid":null},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Akul Singh","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, PA, USA"],"raw_orcid":"https://orcid.org/0009-0004-4306-8344","affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, PA, USA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5117427924","display_name":"Saisha Kartik","orcid":null},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Saisha Kartik","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, PA, USA"],"raw_orcid":"https://orcid.org/0009-0009-5405-9669","affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, PA, USA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5117427925","display_name":"Audi Lin","orcid":null},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Audi Lin","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, PA, USA"],"raw_orcid":"https://orcid.org/0009-0008-6988-4541","affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, PA, USA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5017938244","display_name":"Tarana Laroia","orcid":"https://orcid.org/0009-0003-8346-8724"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Tarana Laroia","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, PA, USA"],"raw_orcid":"https://orcid.org/0009-0003-8346-8724","affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, PA, USA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5056379112","display_name":"Anthony Rowe","orcid":"https://orcid.org/0000-0003-2332-9450"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Anthony Rowe","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, PA, USA"],"raw_orcid":"https://orcid.org/0000-0003-2332-9450","affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, PA, USA","institution_ids":["https://openalex.org/I74973139"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I74973139"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":3,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"700","last_page":"701"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10715","display_name":"Distributed and Parallel Computing Systems","score":0.6186000108718872,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10715","display_name":"Distributed and Parallel Computing Systems","score":0.6186000108718872,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.5867000222206116,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6707133650779724}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6707133650779724}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3715014.3724366","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3715014.3724366","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 23rd ACM Conference on Embedded Networked Sensor Systems","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education","score":0.5299999713897705}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":1,"referenced_works":["https://openalex.org/W4406266242"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W4391913857","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W4396696052"],"abstract_inverted_index":{"As":[0],"Artificial":[1],"Intelligence":[2],"(AI)":[3],"and":[4,26,41,66,89,143,147],"eXtended":[5],"Reality":[6],"(XR)":[7],"evolve,":[8],"integrating":[9],"them":[10],"effectively":[11],"remains":[12],"a":[13,54,70,96,118,121,127],"challenge.":[14],"Although":[15],"multimodal":[16],"large":[17],"language":[18],"models":[19,45],"(MLLMs)":[20],"offer":[21],"powerful":[22],"reasoning":[23,65],"over":[24],"text":[25],"images,":[27],"they":[28],"lack":[29],"an":[30,75,111],"inherent":[31],"understanding":[32],"of":[33,85,123],"3D":[34,87],"space.":[35],"Additionally,":[36],"XR":[37,60,76,112],"headsets":[38],"are":[39],"resource-constrained":[40],"cannot":[42],"run":[43],"these":[44],"locally.":[46],"To":[47],"address":[48],"this":[49],"gap,":[50],"we":[51],"introduce":[52],"XaiR,":[53],"system":[55,132],"that":[56,116],"integrates":[57],"MLLMs":[58,102],"with":[59],"to":[61,103],"enable":[62],"AI-driven":[63],"spatial":[64,80],"interaction.":[67],"XaiR":[68],"employs":[69],"client-server":[71],"architecture":[72],"in":[73,140],"which":[74],"headset":[77],"(client)":[78],"captures":[79],"data,":[81],"generates":[82],"2D":[83],"snapshots":[84],"the":[86],"environment,":[88],"renders":[90],"augmented":[91],"reality":[92],"(AR)":[93],"content,":[94],"while":[95],"remote":[97],"server":[98],"runs":[99],"multiple":[100],"parallel":[101],"generate":[104],"contextually":[105],"aware":[106],"responses.":[107],"Our":[108],"demo":[109],"showcases":[110],"cognitive":[113],"assistant":[114],"application":[115],"guides":[117],"user":[119,135],"through":[120],"series":[122],"instructions.":[124],"Deployed":[125],"on":[126],"mobile":[128],"AR":[129],"headset,":[130],"our":[131],"dynamically":[133],"interprets":[134],"actions,":[136],"tracks":[137],"task":[138],"progress":[139],"real":[141],"time,":[142],"provides":[144],"textual":[145],"feedback":[146],"AR-guided":[148],"assistance.":[149]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":1}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}