{"id":"https://openalex.org/W4415938696","doi":"https://doi.org/10.48550/arxiv.2511.00730","title":"Teaching LLMs to See and Guide: Context-Aware Real-Time Assistance in Augmented Reality","display_name":"Teaching LLMs to See and Guide: Context-Aware Real-Time Assistance in Augmented Reality","alternate_titles":[],"publication_year":2025,"publication_date":"2025-11-01","ids":{"openalex":"https://openalex.org/W4415938696","doi":"https://doi.org/10.48550/arxiv.2511.00730"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2511.00730","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2511.00730","pdf_url":"https://arxiv.org/pdf/2511.00730","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"parent_works":[],"lineage":["https://openalex.org/W4415938696"],"child_works_count":0,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2511.00730","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5114467113","display_name":"Mahya Qorbani","orcid":"https://orcid.org/0000-0002-5653-8673","observed_orcids":["https://orcid.org/0000-0002-5653-8673"]},"institutions":[{"id":"https://openalex.org/I130701444","display_name":"Georgia Institute of Technology","ror":"https://ror.org/01zkghx44","country_code":"US","type":"education","lineage":["https://openalex.org/I130701444"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Qorbani, Mahya","raw_affiliation_strings":["George W. Woodruff School of Mechanical Engineering , Georgia Institute of Technology , Atlanta , GA , USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"George W. Woodruff School of Mechanical Engineering , Georgia Institute of Technology , Atlanta , GA , USA","institution_ids":["https://openalex.org/I130701444"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5072570845","display_name":"Kamran Paynabar","orcid":"https://orcid.org/0000-0002-6906-3611","observed_orcids":["https://orcid.org/0000-0002-6906-3611"]},"institutions":[{"id":"https://openalex.org/I130701444","display_name":"Georgia Institute of Technology","ror":"https://ror.org/01zkghx44","country_code":"US","type":"education","lineage":["https://openalex.org/I130701444"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Paynabar, Kamran","raw_affiliation_strings":["George W. Woodruff School of Mechanical Engineering , Georgia Institute of Technology , Atlanta , GA , USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"George W. Woodruff School of Mechanical Engineering , Georgia Institute of Technology , Atlanta , GA , USA","institution_ids":["https://openalex.org/I130701444"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5017863793","display_name":"Mohsen Moghaddam","orcid":"https://orcid.org/0000-0002-3201-6010","observed_orcids":["https://orcid.org/0000-0002-3201-6010"]},"institutions":[{"id":"https://openalex.org/I130701444","display_name":"Georgia Institute of Technology","ror":"https://ror.org/01zkghx44","country_code":"US","type":"education","lineage":["https://openalex.org/I130701444"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Moghaddam, Mohsen","raw_affiliation_strings":["School of Industrial and Systems Engineering , Georgia Institute of Technology , Atlanta , GA , USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Industrial and Systems Engineering , Georgia Institute of Technology , Atlanta , GA , USA","institution_ids":["https://openalex.org/I130701444"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I130701444"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":9,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_pulled":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10888","display_name":"Augmented Reality Applications","score":0.7954000234603882,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10888","display_name":"Augmented Reality Applications","score":0.7954000234603882,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.16060000658035278,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.009800000116229057,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.8799999952316284},{"id":"https://openalex.org/keywords/augmented-reality","display_name":"augmented reality","score":0.968999981880188},{"id":"https://openalex.org/keywords/virtual-reality","display_name":"virtual reality","score":0.8669999837875366},{"id":"https://openalex.org/keywords/context-aware-assistance","display_name":"context-aware assistance","score":0.7260000109672546},{"id":"https://openalex.org/keywords/real-time-question-answering","display_name":"real-time question answering","score":0.8709999918937683},{"id":"https://openalex.org/keywords/multimodal-context","display_name":"multimodal context","score":0.6610000133514404},{"id":"https://openalex.org/keywords/gaze-data","display_name":"gaze data","score":0.6679999828338623},{"id":"https://openalex.org/keywords/task-progression","display_name":"task progression","score":0.5090000033378601},{"id":"https://openalex.org/keywords/industrial-training","display_name":"industrial training","score":0.878000020980835}],"concepts":[{"id":"https://openalex.org/C153715457","wikidata":"https://www.wikidata.org/wiki/Q254183","display_name":"Augmented reality","level":2,"score":0.8126000165939331},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.7555999755859375},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.6832000017166138},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.660099983215332},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.642300009727478},{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.6197999715805054},{"id":"https://openalex.org/C2780226545","wikidata":"https://www.wikidata.org/wiki/Q6888030","display_name":"Modality (human\u2013computer interaction)","level":2,"score":0.47920000553131104},{"id":"https://openalex.org/C175154964","wikidata":"https://www.wikidata.org/wiki/Q380077","display_name":"Task analysis","level":3,"score":0.4284000098705292},{"id":"https://openalex.org/C2780910867","wikidata":"https://www.wikidata.org/wiki/Q1952416","display_name":"Multimodality","level":2,"score":0.4018999934196472},{"id":"https://openalex.org/C194969405","wikidata":"https://www.wikidata.org/wiki/Q170519","display_name":"Virtual reality","level":2,"score":0.39879998564720154},{"id":"https://openalex.org/C135641252","wikidata":"https://www.wikidata.org/wiki/Q738567","display_name":"Multimodal interaction","level":2,"score":0.39329999685287476},{"id":"https://openalex.org/C49774154","wikidata":"https://www.wikidata.org/wiki/Q131765","display_name":"Multimedia","level":1,"score":0.3450999855995178},{"id":"https://openalex.org/C56739046","wikidata":"https://www.wikidata.org/wiki/Q192060","display_name":"Knowledge management","level":1,"score":0.3328999876976013},{"id":"https://openalex.org/C71611378","wikidata":"https://www.wikidata.org/wiki/Q5165191","display_name":"Contextual design","level":3,"score":0.32600000500679016},{"id":"https://openalex.org/C77618280","wikidata":"https://www.wikidata.org/wiki/Q1155772","display_name":"Scheme (mathematics)","level":2,"score":0.32089999318122864},{"id":"https://openalex.org/C183322885","wikidata":"https://www.wikidata.org/wiki/Q17007702","display_name":"Context model","level":3,"score":0.3188999891281128},{"id":"https://openalex.org/C48103436","wikidata":"https://www.wikidata.org/wiki/Q599031","display_name":"State (computer science)","level":2,"score":0.3183000087738037},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.29820001125335693},{"id":"https://openalex.org/C107953548","wikidata":"https://www.wikidata.org/wiki/Q1172433","display_name":"Wired glove","level":3,"score":0.2897999882698059},{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.2639000117778778},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.2619999945163727},{"id":"https://openalex.org/C135355979","wikidata":"https://www.wikidata.org/wiki/Q5165193","display_name":"Contextual inquiry","level":2,"score":0.25870001316070557}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2511.00730","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2511.00730","pdf_url":"https://arxiv.org/pdf/2511.00730","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2511.00730","doi":"https://doi.org/10.48550/arxiv.2511.00730","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2511.00730","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2511.00730","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2511.00730","pdf_url":"https://arxiv.org/pdf/2511.00730","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G1158794305","display_name":"FW-HTF-R: Fostering Learning and Adaptability of Future Manufacturing Workers with Intelligent Extended Reality (IXR)","funder_award_id":"2128743","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4415938696.pdf","grobid_xml":"https://content.openalex.org/works/W4415938696.grobid-xml"},"referenced_works_count":50,"referenced_works":["https://openalex.org/W2101105183","https://openalex.org/W2154652894","https://openalex.org/W2606829066","https://openalex.org/W2741672218","https://openalex.org/W2992509920","https://openalex.org/W3100712674","https://openalex.org/W3201345620","https://openalex.org/W4292969683","https://openalex.org/W4385571232","https://openalex.org/W4385572383","https://openalex.org/W4389519254","https://openalex.org/W4390871800","https://openalex.org/W4391987972","https://openalex.org/W4392902876","https://openalex.org/W4393404844","https://openalex.org/W4396832611","https://openalex.org/W4399266923","https://openalex.org/W4399354028","https://openalex.org/W4399554057","https://openalex.org/W4400316914","https://openalex.org/W4401043863","https://openalex.org/W4402671800","https://openalex.org/W4402684150","https://openalex.org/W4402721904","https://openalex.org/W4403570354","https://openalex.org/W4403582514","https://openalex.org/W4403995458","https://openalex.org/W4404585558","https://openalex.org/W4404818086","https://openalex.org/W4405253182","https://openalex.org/W4405304093","https://openalex.org/W4405399835","https://openalex.org/W4405455580","https://openalex.org/W4405924600","https://openalex.org/W4407100023","https://openalex.org/W4407385160","https://openalex.org/W4408135318","https://openalex.org/W4408147583","https://openalex.org/W4408954469","https://openalex.org/W4409351777","https://openalex.org/W4412889641","https://openalex.org/W4412939656","https://openalex.org/W4413157121","https://openalex.org/W4413187589","https://openalex.org/W4413925698","https://openalex.org/W4414463795","https://openalex.org/W7084032685","https://openalex.org/W7120088840","https://openalex.org/W7133221203","https://openalex.org/W7133239970"],"related_works":["https://openalex.org/W4387800464","https://openalex.org/W7148479892","https://openalex.org/W7133927214","https://openalex.org/W4414564653","https://openalex.org/W7154420852","https://openalex.org/W7164946119","https://openalex.org/W4406747606","https://openalex.org/W4408102244","https://openalex.org/W4409747316","https://openalex.org/W4387271809"],"abstract_inverted_index":{"The":[0],"growing":[1],"adoption":[2],"of":[3,42,159,166],"augmented":[4],"and":[5,9,15,35,49,57,94,157,177,181],"virtual":[6],"reality":[7],"(AR":[8],"VR)":[10],"technologies":[11],"in":[12,60],"industrial":[13,179],"training":[14,180],"on-the-job":[16],"assistance":[17,174],"has":[18],"created":[19],"new":[20],"opportunities":[21],"for":[22,101,175],"intelligent,":[23],"context-aware":[24,77],"support":[25],"systems.":[26],"As":[27],"workers":[28],"perform":[29],"complex":[30],"tasks":[31],"guided":[32],"by":[33],"AR":[34,176],"VR,":[36],"these":[37],"devices":[38],"capture":[39],"rich":[40],"streams":[41],"multimodal":[43,151,168],"data,":[44],"including":[45],"gaze,":[46],"hand":[47,90],"actions,":[48,91],"task":[50,58,92,134],"progression,":[51],"that":[52,83,149],"can":[53],"reveal":[54],"user":[55],"intent":[56],"state":[59],"real":[61],"time.":[62],"Leveraging":[63],"this":[64,72],"information":[65],"effectively":[66],"remains":[67],"a":[68,76,98],"major":[69],"challenge.":[70],"In":[71],"work,":[73],"we":[74,112,136],"present":[75],"large":[78],"language":[79],"model":[80,120],"(LLM)":[81],"assistant":[82],"integrates":[84],"diverse":[85],"data":[86],"modalities,":[87],"such":[88],"as":[89],"steps,":[93],"dialogue":[95],"history,":[96],"into":[97],"unified":[99],"framework":[100],"real-time":[102],"question":[103],"answering.":[104],"To":[105],"systematically":[106],"study":[107],"how":[108,138],"context":[109,152],"influences":[110],"performance,":[111],"introduce":[113],"an":[114],"incremental":[115],"prompting":[116],"framework,":[117],"where":[118],"each":[119,139],"version":[121],"receives":[122],"progressively":[123],"richer":[124],"contextual":[125],"inputs.":[126],"Using":[127],"the":[128,143,155,164],"HoloAssist":[129],"dataset,":[130],"which":[131],"records":[132],"AR-guided":[133],"executions,":[135],"evaluate":[137],"modality":[140],"contributes":[141],"to":[142,170],"assistant's":[144],"effectiveness.":[145],"Our":[146],"experiments":[147],"show":[148],"incorporating":[150],"significantly":[153],"improves":[154],"accuracy":[156],"relevance":[158],"responses.":[160],"These":[161],"findings":[162],"highlight":[163],"potential":[165],"LLM-driven":[167],"integration":[169],"enable":[171],"adaptive,":[172],"intuitive":[173],"VR-based":[178],"assistance.":[182]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W4415938696","counts_by_year":[{"year":2026,"cited_by_count":9}],"updated_date":"2026-11-16T12:05:36.265796","created_date":"2025-11-05T00:00:00"}
