{"id":"https://openalex.org/W4403318118","doi":"https://doi.org/10.1145/3672539.3686776","title":"WatchThis: A Wearable Point-and-Ask Interface powered by Vision-Language Models for Contextual Queries","display_name":"WatchThis: A Wearable Point-and-Ask Interface powered by Vision-Language Models for Contextual Queries","publication_year":2024,"publication_date":"2024-10-11","ids":{"openalex":"https://openalex.org/W4403318118","doi":"https://doi.org/10.1145/3672539.3686776"},"language":"en","primary_location":{"id":"doi:10.1145/3672539.3686776","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3672539.3686776","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"The 37th Annual ACM Symposium on User Interface Software and Technology","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5061556449","display_name":"Cathy Mengying Fang","orcid":"https://orcid.org/0000-0003-4684-7058"},"institutions":[{"id":"https://openalex.org/I4210142372","display_name":"Human Media","ror":"https://ror.org/04072nk43","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210142372"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Cathy Mengying Fang","raw_affiliation_strings":["MIT Media Lab, United States"],"raw_orcid":"https://orcid.org/0000-0003-4684-7058","affiliations":[{"raw_affiliation_string":"MIT Media Lab, United States","institution_ids":["https://openalex.org/I4210142372"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5053100310","display_name":"Patrick Chwalek","orcid":"https://orcid.org/0000-0002-4353-7161"},"institutions":[{"id":"https://openalex.org/I4210142372","display_name":"Human Media","ror":"https://ror.org/04072nk43","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210142372"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Patrick Chwalek","raw_affiliation_strings":["MIT Media Lab, United States"],"raw_orcid":"https://orcid.org/0000-0002-4353-7161","affiliations":[{"raw_affiliation_string":"MIT Media Lab, United States","institution_ids":["https://openalex.org/I4210142372"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5023548448","display_name":"Quincy Kuang","orcid":"https://orcid.org/0000-0002-6689-2078"},"institutions":[{"id":"https://openalex.org/I4210142372","display_name":"Human Media","ror":"https://ror.org/04072nk43","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210142372"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Quincy Kuang","raw_affiliation_strings":["MIT Media Lab, United States"],"raw_orcid":"https://orcid.org/0000-0002-6689-2078","affiliations":[{"raw_affiliation_string":"MIT Media Lab, United States","institution_ids":["https://openalex.org/I4210142372"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5081457786","display_name":"Pattie Maes","orcid":"https://orcid.org/0000-0002-7722-6038"},"institutions":[{"id":"https://openalex.org/I4210142372","display_name":"Human Media","ror":"https://ror.org/04072nk43","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210142372"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Pattie Maes","raw_affiliation_strings":["MIT Media Lab, United States"],"raw_orcid":"https://orcid.org/0000-0002-7722-6038","affiliations":[{"raw_affiliation_string":"MIT Media Lab, United States","institution_ids":["https://openalex.org/I4210142372"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.4375,"has_fulltext":false,"cited_by_count":2,"citation_normalized_percentile":{"value":0.63149177,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":91,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"4"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10888","display_name":"Augmented Reality Applications","score":0.9807000160217285,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10888","display_name":"Augmented Reality Applications","score":0.9807000160217285,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10627","display_name":"Advanced Image and Video Retrieval Techniques","score":0.9801999926567078,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11707","display_name":"Gaze Tracking and Assistive Technology","score":0.9735000133514404,"subfield":{"id":"https://openalex.org/subfields/1709","display_name":"Human-Computer Interaction"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/ask-price","display_name":"Ask price","score":0.9035024642944336},{"id":"https://openalex.org/keywords/wearable-computer","display_name":"Wearable computer","score":0.7623077630996704},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7306542992591858},{"id":"https://openalex.org/keywords/point","display_name":"Point (geometry)","score":0.6587924957275391},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.6266475319862366},{"id":"https://openalex.org/keywords/interface","display_name":"Interface (matter)","score":0.5340838432312012},{"id":"https://openalex.org/keywords/wearable-technology","display_name":"Wearable technology","score":0.5283138155937195},{"id":"https://openalex.org/keywords/user-interface","display_name":"User interface","score":0.445600688457489},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3292805254459381},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.08904898166656494},{"id":"https://openalex.org/keywords/embedded-system","display_name":"Embedded system","score":0.07261109352111816}],"concepts":[{"id":"https://openalex.org/C90329073","wikidata":"https://www.wikidata.org/wiki/Q914232","display_name":"Ask price","level":2,"score":0.9035024642944336},{"id":"https://openalex.org/C150594956","wikidata":"https://www.wikidata.org/wiki/Q1334829","display_name":"Wearable computer","level":2,"score":0.7623077630996704},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7306542992591858},{"id":"https://openalex.org/C28719098","wikidata":"https://www.wikidata.org/wiki/Q44946","display_name":"Point (geometry)","level":2,"score":0.6587924957275391},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.6266475319862366},{"id":"https://openalex.org/C113843644","wikidata":"https://www.wikidata.org/wiki/Q901882","display_name":"Interface (matter)","level":4,"score":0.5340838432312012},{"id":"https://openalex.org/C54290928","wikidata":"https://www.wikidata.org/wiki/Q4845080","display_name":"Wearable technology","level":3,"score":0.5283138155937195},{"id":"https://openalex.org/C89505385","wikidata":"https://www.wikidata.org/wiki/Q47146","display_name":"User interface","level":2,"score":0.445600688457489},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3292805254459381},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.08904898166656494},{"id":"https://openalex.org/C149635348","wikidata":"https://www.wikidata.org/wiki/Q193040","display_name":"Embedded system","level":1,"score":0.07261109352111816},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.0},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C129307140","wikidata":"https://www.wikidata.org/wiki/Q6795880","display_name":"Maximum bubble pressure method","level":3,"score":0.0},{"id":"https://openalex.org/C136264566","wikidata":"https://www.wikidata.org/wiki/Q159810","display_name":"Economy","level":1,"score":0.0},{"id":"https://openalex.org/C157915830","wikidata":"https://www.wikidata.org/wiki/Q2928001","display_name":"Bubble","level":2,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3672539.3686776","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3672539.3686776","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"The 37th Annual ACM Symposium on User Interface Software and Technology","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":5,"referenced_works":["https://openalex.org/W2135643110","https://openalex.org/W2136994117","https://openalex.org/W3031588686","https://openalex.org/W3031942796","https://openalex.org/W4388127189"],"related_works":["https://openalex.org/W3090300519","https://openalex.org/W2514492205","https://openalex.org/W4250401876","https://openalex.org/W2943851981","https://openalex.org/W2566526749","https://openalex.org/W2907667791","https://openalex.org/W3047461507","https://openalex.org/W3126390843","https://openalex.org/W4245880644","https://openalex.org/W4312415459"],"abstract_inverted_index":{"This":[0,81],"paper":[1,107],"introduces":[2],"WatchThis,":[3],"a":[4,45,54,58,99],"novel":[5],"wearable":[6,55],"device":[7],"that":[8,62,88],"enables":[9],"natural":[10,79],"language":[11],"interactions":[12],"with":[13,57],"real-world":[14],"objects":[15],"and":[16,38,119,126,137],"environments":[17],"through":[18],"pointing":[19],"gestures.":[20],"Building":[21],"upon":[22],"previous":[23],"work":[24,131],"in":[25,33,78,139],"gesture-based":[26],"computing":[27],"interfaces,":[28],"WatchThis":[29],"leverages":[30],"recent":[31],"advancements":[32],"Large":[34],"Language":[35,40],"Models":[36,41],"(LLM)":[37],"Vision":[39],"(VLM)":[42],"to":[43,72,133],"create":[44],"hands-free,":[46],"contextual":[47],"querying":[48],"system.":[49],"The":[50,106],"prototype":[51],"consists":[52],"of":[53,66,85,111],"watch":[56],"rotating,":[59],"flip-up":[60],"camera":[61],"captures":[63],"the":[64,94,128],"area":[65],"interest":[67],"when":[68],"pointing,":[69],"allowing":[70],"users":[71],"ask":[73],"questions":[74],"about":[75],"their":[76],"surroundings":[77],"language.":[80],"design":[82],"addresses":[83],"limitations":[84],"existing":[86],"systems":[87],"require":[89],"specific":[90],"commands":[91],"or":[92],"occupy":[93],"hands,":[95],"while":[96],"also":[97],"maintaining":[98],"non-discrete":[100],"form":[101],"factor":[102],"for":[103],"social":[104],"awareness.":[105],"explores":[108],"various":[109],"applications":[110],"this":[112,130],"point-and-ask":[113],"interaction,":[114],"including":[115],"object":[116],"identification,":[117],"translation,":[118],"instruction":[120],"queries.":[121],"By":[122],"utilizing":[123],"off-the-shelf":[124],"components":[125],"open-sourcing":[127],"design,":[129],"aims":[132],"facilitate":[134],"further":[135],"research":[136],"development":[138],"wearable,":[140],"AI-enabled":[141],"interaction":[142],"paradigms.":[143]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":1}],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-10-10T00:00:00"}