{"id":"https://openalex.org/W7154072686","doi":"https://doi.org/10.1145/3772318.3790946","title":"How Humans Naturally Refer to Targets: Understanding Multimodal Instruction Patterns in Human\u2013Robot Interaction","display_name":"How Humans Naturally Refer to Targets: Understanding Multimodal Instruction Patterns in Human\u2013Robot Interaction","publication_year":2026,"publication_date":"2026-04-13","ids":{"openalex":"https://openalex.org/W7154072686","doi":"https://doi.org/10.1145/3772318.3790946"},"language":"en","primary_location":{"id":"doi:10.1145/3772318.3790946","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3772318.3790946","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2026 CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3772318.3790946","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5048670847","display_name":"Lesong Jia","orcid":"https://orcid.org/0000-0002-3950-0905"},"institutions":[{"id":"https://openalex.org/I170201317","display_name":"University of Pittsburgh","ror":"https://ror.org/01an3r305","country_code":"US","type":"education","lineage":["https://openalex.org/I170201317"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Lesong Jia","raw_affiliation_strings":["University of Pittsburgh, Pittsburgh, Pennsylvania, USA"],"raw_orcid":"https://orcid.org/0000-0002-3950-0905","affiliations":[{"raw_affiliation_string":"University of Pittsburgh, Pittsburgh, Pennsylvania, USA","institution_ids":["https://openalex.org/I170201317"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Makayla Chang","orcid":"https://orcid.org/0009-0001-6507-8218"},"institutions":[{"id":"https://openalex.org/I170201317","display_name":"University of Pittsburgh","ror":"https://ror.org/01an3r305","country_code":"US","type":"education","lineage":["https://openalex.org/I170201317"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Makayla Chang","raw_affiliation_strings":["University of Pittsburgh, Pittsburgh, Pennsylvania, USA"],"raw_orcid":"https://orcid.org/0009-0001-6507-8218","affiliations":[{"raw_affiliation_string":"University of Pittsburgh, Pittsburgh, Pennsylvania, USA","institution_ids":["https://openalex.org/I170201317"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133548748","display_name":"Yu Liu","orcid":"https://orcid.org/0009-0005-7153-504X"},"institutions":[{"id":"https://openalex.org/I79272384","display_name":"University of Maryland, Baltimore County","ror":"https://ror.org/02qskvh78","country_code":"US","type":"education","lineage":["https://openalex.org/I79272384"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yu Liu","raw_affiliation_strings":["University of Maryland, Baltimore County, Baltimore, Maryland, USA"],"raw_orcid":"https://orcid.org/0009-0005-7153-504X","affiliations":[{"raw_affiliation_string":"University of Maryland, Baltimore County, Baltimore, Maryland, USA","institution_ids":["https://openalex.org/I79272384"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5027093356","display_name":"Na Du","orcid":"https://orcid.org/0000-0002-4383-2451"},"institutions":[{"id":"https://openalex.org/I170201317","display_name":"University of Pittsburgh","ror":"https://ror.org/01an3r305","country_code":"US","type":"education","lineage":["https://openalex.org/I170201317"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Na Du","raw_affiliation_strings":["School of Computing and Information, University of Pittsburgh, Pittsburgh, Pennsylvania, USA"],"raw_orcid":"https://orcid.org/0000-0002-4383-2451","affiliations":[{"raw_affiliation_string":"School of Computing and Information, University of Pittsburgh, Pittsburgh, Pennsylvania, USA","institution_ids":["https://openalex.org/I170201317"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"16"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10709","display_name":"Social Robot Interaction and HRI","score":0.51910001039505,"subfield":{"id":"https://openalex.org/subfields/3207","display_name":"Social Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10709","display_name":"Social Robot Interaction and HRI","score":0.51910001039505,"subfield":{"id":"https://openalex.org/subfields/3207","display_name":"Social Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11707","display_name":"Gaze Tracking and Assistive Technology","score":0.12389999628067017,"subfield":{"id":"https://openalex.org/subfields/1709","display_name":"Human-Computer Interaction"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11431","display_name":"Action Observation and Synchronization","score":0.07649999856948853,"subfield":{"id":"https://openalex.org/subfields/3207","display_name":"Social Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/referent","display_name":"Referent","score":0.7541000247001648},{"id":"https://openalex.org/keywords/gaze","display_name":"Gaze","score":0.7087000012397766},{"id":"https://openalex.org/keywords/multimodal-interaction","display_name":"Multimodal interaction","score":0.590499997138977},{"id":"https://openalex.org/keywords/human\u2013robot-interaction","display_name":"Human\u2013robot interaction","score":0.5810999870300293},{"id":"https://openalex.org/keywords/joint-attention","display_name":"Joint attention","score":0.5005999803543091},{"id":"https://openalex.org/keywords/robot","display_name":"Robot","score":0.4602000117301941},{"id":"https://openalex.org/keywords/interface","display_name":"Interface (matter)","score":0.4350000023841858}],"concepts":[{"id":"https://openalex.org/C2777096784","wikidata":"https://www.wikidata.org/wiki/Q3826351","display_name":"Referent","level":2,"score":0.7541000247001648},{"id":"https://openalex.org/C2779916870","wikidata":"https://www.wikidata.org/wiki/Q14467155","display_name":"Gaze","level":2,"score":0.7087000012397766},{"id":"https://openalex.org/C135641252","wikidata":"https://www.wikidata.org/wiki/Q738567","display_name":"Multimodal interaction","level":2,"score":0.590499997138977},{"id":"https://openalex.org/C145460709","wikidata":"https://www.wikidata.org/wiki/Q859951","display_name":"Human\u2013robot interaction","level":3,"score":0.5810999870300293},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5676000118255615},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.5354999899864197},{"id":"https://openalex.org/C137878579","wikidata":"https://www.wikidata.org/wiki/Q9636076","display_name":"Joint attention","level":3,"score":0.5005999803543091},{"id":"https://openalex.org/C90509273","wikidata":"https://www.wikidata.org/wiki/Q11012","display_name":"Robot","level":2,"score":0.4602000117301941},{"id":"https://openalex.org/C113843644","wikidata":"https://www.wikidata.org/wiki/Q901882","display_name":"Interface (matter)","level":4,"score":0.4350000023841858},{"id":"https://openalex.org/C2780910867","wikidata":"https://www.wikidata.org/wiki/Q1952416","display_name":"Multimodality","level":2,"score":0.412200003862381},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.4090999960899353},{"id":"https://openalex.org/C56461940","wikidata":"https://www.wikidata.org/wiki/Q970687","display_name":"Eye tracking","level":2,"score":0.3930000066757202},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.3686000108718872},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.36070001125335693},{"id":"https://openalex.org/C175154964","wikidata":"https://www.wikidata.org/wiki/Q380077","display_name":"Task analysis","level":3,"score":0.3508000075817108},{"id":"https://openalex.org/C59656382","wikidata":"https://www.wikidata.org/wiki/Q191536","display_name":"Conjunction (astronomy)","level":2,"score":0.35030001401901245},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3465999960899353},{"id":"https://openalex.org/C46312422","wikidata":"https://www.wikidata.org/wiki/Q11024","display_name":"Communication","level":1,"score":0.3395000100135803},{"id":"https://openalex.org/C2780660688","wikidata":"https://www.wikidata.org/wiki/Q25052564","display_name":"Multimodal learning","level":2,"score":0.3070000112056732},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.3012000024318695},{"id":"https://openalex.org/C207347870","wikidata":"https://www.wikidata.org/wiki/Q371174","display_name":"Gesture","level":2,"score":0.2799000144004822},{"id":"https://openalex.org/C4441509","wikidata":"https://www.wikidata.org/wiki/Q6418787","display_name":"Multimodal therapy","level":2,"score":0.2644999921321869},{"id":"https://openalex.org/C2777877512","wikidata":"https://www.wikidata.org/wiki/Q1116097","display_name":"Common ground","level":2,"score":0.2506999969482422}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3772318.3790946","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3772318.3790946","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2026 CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"},{"id":"pmh:doi:10.13016/m2e8a3-ssyt","is_oa":true,"landing_page_url":"http://hdl.handle.net/11603/42900","pdf_url":null,"source":{"id":"https://openalex.org/S4406922384","display_name":"Open MIND","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Text"}],"best_oa_location":{"id":"doi:10.1145/3772318.3790946","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3772318.3790946","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2026 CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"},"sustainable_development_goals":[{"score":0.7047696709632874,"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":87,"referenced_works":["https://openalex.org/W164263582","https://openalex.org/W1769766843","https://openalex.org/W1805396838","https://openalex.org/W1969214042","https://openalex.org/W1979660864","https://openalex.org/W1998524130","https://openalex.org/W2016565202","https://openalex.org/W2022993124","https://openalex.org/W2033035423","https://openalex.org/W2047539061","https://openalex.org/W2067944299","https://openalex.org/W2072500831","https://openalex.org/W2076310157","https://openalex.org/W2096175621","https://openalex.org/W2099886412","https://openalex.org/W2110930288","https://openalex.org/W2117669053","https://openalex.org/W2137070227","https://openalex.org/W2159941522","https://openalex.org/W2161592246","https://openalex.org/W2264742718","https://openalex.org/W2417334932","https://openalex.org/W2490742492","https://openalex.org/W2506096269","https://openalex.org/W2566724781","https://openalex.org/W2617711795","https://openalex.org/W2764065835","https://openalex.org/W2771496173","https://openalex.org/W2773729621","https://openalex.org/W2795730217","https://openalex.org/W2886206057","https://openalex.org/W2890177301","https://openalex.org/W2895815507","https://openalex.org/W2898277016","https://openalex.org/W2899868091","https://openalex.org/W2910779592","https://openalex.org/W2910814616","https://openalex.org/W3004121726","https://openalex.org/W3024383962","https://openalex.org/W3029930070","https://openalex.org/W3046443127","https://openalex.org/W3102151175","https://openalex.org/W3115478544","https://openalex.org/W3159367349","https://openalex.org/W3163078982","https://openalex.org/W3163086318","https://openalex.org/W3166417756","https://openalex.org/W3194424775","https://openalex.org/W3197501724","https://openalex.org/W3217183157","https://openalex.org/W4200328550","https://openalex.org/W4225085515","https://openalex.org/W4251455517","https://openalex.org/W4280611089","https://openalex.org/W4284674177","https://openalex.org/W4297670218","https://openalex.org/W4299627282","https://openalex.org/W4309604219","https://openalex.org/W4313547852","https://openalex.org/W4320921617","https://openalex.org/W4322825287","https://openalex.org/W4377012710","https://openalex.org/W4383066118","https://openalex.org/W4385489615","https://openalex.org/W4386249396","https://openalex.org/W4387376888","https://openalex.org/W4389266910","https://openalex.org/W4390660258","https://openalex.org/W4392489696","https://openalex.org/W4392661554","https://openalex.org/W4396832435","https://openalex.org/W4396832611","https://openalex.org/W4399105147","https://openalex.org/W4400904804","https://openalex.org/W4401414646","https://openalex.org/W4402521074","https://openalex.org/W4404340519","https://openalex.org/W4408117716","https://openalex.org/W4408222737","https://openalex.org/W4409064701","https://openalex.org/W4409720015","https://openalex.org/W4409767753","https://openalex.org/W4409884910","https://openalex.org/W4410297125","https://openalex.org/W4410503323","https://openalex.org/W6948088389","https://openalex.org/W7077959027"],"related_works":[],"abstract_inverted_index":{"Current":[0],"multimodal":[1,55,98,141],"instruction-recognition":[2],"algorithms":[3],"in":[4,18,36,108],"human-robot":[5],"interaction,":[6],"developed":[7],"largely":[8],"from":[9,58],"a":[10,26,64],"purely":[11],"technical":[12],"perspective,":[13],"remain":[14],"rigid":[15],"and":[16,45,76,89,116,136,153],"incomplete":[17],"their":[19],"use":[20],"of":[21,29],"human":[22],"communicative":[23],"cues.":[24],"Therefore,":[25],"full":[27],"understanding":[28],"how":[30],"humans":[31],"naturally":[32,62],"refer":[33],"to":[34,40,43],"targets":[35],"interaction":[37],"is":[38],"central":[39],"enabling":[41],"robots":[42],"interpret":[44],"act":[46],"on":[47],"user":[48],"instructions.":[49],"To":[50,94],"investigate":[51],"this,":[52],"we":[53,70],"collected":[54],"behavior":[56],"data":[57],"30":[59],"participants":[60],"who":[61],"instructed":[63],"robot":[65],"for":[66,123,150],"household":[67],"tasks":[68],"while":[69,111],"systematically":[71],"varied":[72,133],"target":[73,145],"distance,":[74],"direction,":[75,146],"local":[77,137],"referent":[78,138],"complexity.":[79],"Our":[80],"results":[81],"show":[82],"that":[83,130],"speech":[84,117,131],"instructions":[85],"were":[86],"often":[87],"vague":[88],"lacked":[90],"explicit":[91],"target-position":[92],"information.":[93],"resolve":[95],"this":[96],"ambiguity,":[97],"cues":[99],"are":[100],"essential:":[101],"gaze":[102],"direction":[103],"provides":[104],"an":[105],"order-of-magnitude":[106],"improvement":[107],"target-localization":[109],"accuracy,":[110],"hand":[112],"pointing,":[113],"head":[114],"turns,":[115],"onset":[118],"offer":[119],"reliable":[120],"temporal":[121],"anchors":[122],"identifying":[124],"target-directed":[125],"gaze.":[126],"We":[127],"also":[128],"found":[129],"patterns":[132],"with":[134,144],"distance":[135],"complexity,":[139],"whereas":[140],"behaviors":[142],"shifted":[143],"underscoring":[147],"the":[148],"need":[149],"context-adaptive":[151],"recognition":[152],"interface":[154],"design.":[155]},"counts_by_year":[],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2026-04-14T00:00:00"}