{"id":"https://openalex.org/W4409748212","doi":"https://doi.org/10.1145/3706598.3714259","title":"AI-Instruments: Embodying Prompts as Instruments to Abstract & Reflect Graphical Interface Commands as General-Purpose Tools","display_name":"AI-Instruments: Embodying Prompts as Instruments to Abstract & Reflect Graphical Interface Commands as General-Purpose Tools","publication_year":2025,"publication_date":"2025-04-24","ids":{"openalex":"https://openalex.org/W4409748212","doi":"https://doi.org/10.1145/3706598.3714259"},"language":"en","primary_location":{"id":"doi:10.1145/3706598.3714259","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3706598.3714259","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2025 CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"},"type":"preprint","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3706598.3714259","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5068799512","display_name":"Nathalie Henry Riche","orcid":"https://orcid.org/0000-0003-1759-7512"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Nathalie Riche","raw_affiliation_strings":["Microsoft Research, Redmond, Washington, USA"],"raw_orcid":"https://orcid.org/0000-0003-1759-7512","affiliations":[{"raw_affiliation_string":"Microsoft Research, Redmond, Washington, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5020407163","display_name":"Anna Offenwanger","orcid":"https://orcid.org/0000-0001-9560-201X"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]},{"id":"https://openalex.org/I1294671590","display_name":"Centre National de la Recherche Scientifique","ror":"https://ror.org/02feahw73","country_code":"FR","type":"government","lineage":["https://openalex.org/I1294671590"]},{"id":"https://openalex.org/I277688954","display_name":"Universit\u00e9 Paris-Saclay","ror":"https://ror.org/03xjwb503","country_code":"FR","type":"education","lineage":["https://openalex.org/I277688954"]}],"countries":["FR","US"],"is_corresponding":false,"raw_author_name":"Anna Offenwanger","raw_affiliation_strings":["Microsoft Research, Redmond, Washington, USA and CNRS, Inria, LISN, Universit\u00e9 Paris-Saclay, Orsay, France"],"raw_orcid":"https://orcid.org/0000-0001-9560-201X","affiliations":[{"raw_affiliation_string":"Microsoft Research, Redmond, Washington, USA and CNRS, Inria, LISN, Universit\u00e9 Paris-Saclay, Orsay, France","institution_ids":["https://openalex.org/I277688954","https://openalex.org/I1290206253","https://openalex.org/I1294671590"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5074303572","display_name":"Frederic Gmeiner","orcid":"https://orcid.org/0000-0003-4106-1881"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]},{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Frederic Gmeiner","raw_affiliation_strings":["Microsoft Research, Redmond, Washington, USA and Carnegie Mellon University, Pittsburgh, Pennsylvania, USA"],"raw_orcid":"https://orcid.org/0000-0003-4106-1881","affiliations":[{"raw_affiliation_string":"Microsoft Research, Redmond, Washington, USA and Carnegie Mellon University, Pittsburgh, Pennsylvania, USA","institution_ids":["https://openalex.org/I1290206253","https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5107842900","display_name":"David M. Brown","orcid":null},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"David Brown","raw_affiliation_strings":["Microsoft Research, Redmond, Washington, USA"],"raw_orcid":"https://orcid.org/0000-0002-6013-2713","affiliations":[{"raw_affiliation_string":"Microsoft Research, Redmond, Washington, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5081816373","display_name":"Hugo Romat","orcid":"https://orcid.org/0000-0002-9194-8889"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Hugo Romat","raw_affiliation_strings":["Microsoft, Seattle, Washington, USA"],"raw_orcid":"https://orcid.org/0000-0002-9194-8889","affiliations":[{"raw_affiliation_string":"Microsoft, Seattle, Washington, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5019041283","display_name":"Michel Pahud","orcid":"https://orcid.org/0000-0003-3924-1189"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Michel Pahud","raw_affiliation_strings":["Microsoft Research, Redmond, Washington, USA"],"raw_orcid":"https://orcid.org/0000-0003-3924-1189","affiliations":[{"raw_affiliation_string":"Microsoft Research, Redmond, Washington, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5036819859","display_name":"Nicolai Marquardt","orcid":"https://orcid.org/0000-0002-5473-2448"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Nicolai Marquardt","raw_affiliation_strings":["Microsoft Research, Redmond, Washington, USA"],"raw_orcid":"https://orcid.org/0000-0002-5473-2448","affiliations":[{"raw_affiliation_string":"Microsoft Research, Redmond, Washington, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5069122334","display_name":"Kori Inkpen","orcid":"https://orcid.org/0000-0001-5661-4659"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kori Inkpen","raw_affiliation_strings":["Microsoft Research, Redmond, Washington, USA"],"raw_orcid":"https://orcid.org/0000-0001-5661-4659","affiliations":[{"raw_affiliation_string":"Microsoft Research, Redmond, Washington, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5036357098","display_name":"Ken Hinckley","orcid":"https://orcid.org/0000-0002-4733-4927"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ken Hinckley","raw_affiliation_strings":["Microsoft Research, Redmond, Washington, USA"],"raw_orcid":"https://orcid.org/0000-0002-4733-4927","affiliations":[{"raw_affiliation_string":"Microsoft Research, Redmond, Washington, USA","institution_ids":["https://openalex.org/I1290206253"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":9,"corresponding_author_ids":["https://openalex.org/A5068799512"],"corresponding_institution_ids":["https://openalex.org/I1290206253"],"apc_list":null,"apc_paid":null,"fwci":19.7967,"has_fulltext":true,"cited_by_count":14,"citation_normalized_percentile":{"value":0.99496014,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"18"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10789","display_name":"Interactive and Immersive Displays","score":0.9958000183105469,"subfield":{"id":"https://openalex.org/subfields/1709","display_name":"Human-Computer Interaction"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10789","display_name":"Interactive and Immersive Displays","score":0.9958000183105469,"subfield":{"id":"https://openalex.org/subfields/1709","display_name":"Human-Computer Interaction"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10803","display_name":"Innovative Human-Technology Interaction","score":0.9886999726295471,"subfield":{"id":"https://openalex.org/subfields/1709","display_name":"Human-Computer Interaction"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10672","display_name":"Design Education and Practice","score":0.9805999994277954,"subfield":{"id":"https://openalex.org/subfields/2210","display_name":"Mechanical Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6826695799827576},{"id":"https://openalex.org/keywords/graphical-user-interface","display_name":"Graphical user interface","score":0.6594263315200806},{"id":"https://openalex.org/keywords/interface","display_name":"Interface (matter)","score":0.6407787799835205},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.6392961144447327},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.4424012303352356},{"id":"https://openalex.org/keywords/operating-system","display_name":"Operating system","score":0.16600465774536133}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6826695799827576},{"id":"https://openalex.org/C37789001","wikidata":"https://www.wikidata.org/wiki/Q782543","display_name":"Graphical user interface","level":2,"score":0.6594263315200806},{"id":"https://openalex.org/C113843644","wikidata":"https://www.wikidata.org/wiki/Q901882","display_name":"Interface (matter)","level":4,"score":0.6407787799835205},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.6392961144447327},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.4424012303352356},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.16600465774536133},{"id":"https://openalex.org/C157915830","wikidata":"https://www.wikidata.org/wiki/Q2928001","display_name":"Bubble","level":2,"score":0.0},{"id":"https://openalex.org/C129307140","wikidata":"https://www.wikidata.org/wiki/Q6795880","display_name":"Maximum bubble pressure method","level":3,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1145/3706598.3714259","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3706598.3714259","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2025 CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2502.18736","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2502.18736","pdf_url":"https://arxiv.org/pdf/2502.18736","source":{"id":"https://openalex.org/S4393918464","display_name":"ArXiv.org","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:HAL:hal-05085707v1","is_oa":true,"landing_page_url":"https://hal.science/hal-05085707","pdf_url":"https://hal.science/hal-05085707/document","source":{"id":"https://openalex.org/S4306402512","display_name":"HAL (Le Centre pour la Communication Scientifique Directe)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1294671590","host_organization_name":"Centre National de la Recherche Scientifique","host_organization_lineage":["https://openalex.org/I1294671590"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"CHI 2025 - CHI Conference on Human Factors in Computing Systems, Apr 2025, Yokohama, Japan. pp.1-18, ⟨10.1145/3706598.3714259⟩","raw_type":"Conference papers"}],"best_oa_location":{"id":"doi:10.1145/3706598.3714259","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3706598.3714259","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2025 CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":69,"referenced_works":["https://openalex.org/W1540823594","https://openalex.org/W1607942080","https://openalex.org/W1975397943","https://openalex.org/W1978693696","https://openalex.org/W1982500521","https://openalex.org/W1997447598","https://openalex.org/W2017950175","https://openalex.org/W2031482152","https://openalex.org/W2034079000","https://openalex.org/W2040994656","https://openalex.org/W2047375123","https://openalex.org/W2059216172","https://openalex.org/W2062658884","https://openalex.org/W2073271971","https://openalex.org/W2073336401","https://openalex.org/W2075516808","https://openalex.org/W2085443629","https://openalex.org/W2098832524","https://openalex.org/W2115647291","https://openalex.org/W2121731156","https://openalex.org/W2131597913","https://openalex.org/W2133353349","https://openalex.org/W2136510050","https://openalex.org/W2150489026","https://openalex.org/W2150634379","https://openalex.org/W2153137796","https://openalex.org/W2395063354","https://openalex.org/W2405268431","https://openalex.org/W2611043797","https://openalex.org/W2611474773","https://openalex.org/W2613946508","https://openalex.org/W2796000004","https://openalex.org/W3192495122","https://openalex.org/W3199400376","https://openalex.org/W3212389159","https://openalex.org/W4206008800","https://openalex.org/W4225012671","https://openalex.org/W4225012808","https://openalex.org/W4225116986","https://openalex.org/W4234579550","https://openalex.org/W4237513721","https://openalex.org/W4239943223","https://openalex.org/W4254891401","https://openalex.org/W4312788391","https://openalex.org/W4366548330","https://openalex.org/W4366594418","https://openalex.org/W4377371585","https://openalex.org/W4385682544","https://openalex.org/W4385774603","https://openalex.org/W4387801039","https://openalex.org/W4387801187","https://openalex.org/W4387801427","https://openalex.org/W4387835469","https://openalex.org/W4387993394","https://openalex.org/W4389334940","https://openalex.org/W4390873054","https://openalex.org/W4390874575","https://openalex.org/W4392619076","https://openalex.org/W4393871063","https://openalex.org/W4396828482","https://openalex.org/W4396831794","https://openalex.org/W4396831955","https://openalex.org/W4396831984","https://openalex.org/W4396832002","https://openalex.org/W4396832024","https://openalex.org/W4396832739","https://openalex.org/W4396833147","https://openalex.org/W4400142550","https://openalex.org/W4403334065"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W4391913857","https://openalex.org/W2358668433","https://openalex.org/W2022968056","https://openalex.org/W2993198389","https://openalex.org/W2102284328","https://openalex.org/W1555107492"],"abstract_inverted_index":{"Chat-based":[0],"prompts":[1],"respond":[2],"with":[3],"verbose":[4],"linear-sequential":[5],"texts,":[6],"making":[7],"it":[8],"difficult":[9],"to":[10,65,76,94,123,148],"explore":[11],"and":[12,18,73,97,126,144,150],"refine":[13,98],"ambiguous":[14,53,152],"intents,":[15],"back":[16],"up":[17],"reinterpret,":[19],"or":[20,84],"shift":[21],"directions":[22],"in":[23],"creative":[24],"AI-assisted":[25],"design":[26,67],"work.":[27],"AI-Instruments":[28,91,134],"instead":[29],"embody":[30],"\"prompts\"":[31],"as":[32,43,56,58],"interface":[33],"objects":[34],"via":[35,141],"three":[36],"key":[37],"principles:":[38],"(1)":[39],"Reification":[40],"of":[41,49,52,61,137],"user-intent":[42],"reusable":[44],"direct-manipulation":[45],"instruments;":[46],"(2)":[47],"Reflection":[48],"multiple":[50],"interpretations":[51],"user-intents":[54],"(Reflection-in-intent)":[55],"well":[57],"the":[59],"range":[60],"AI-model":[62],"responses":[63],"(Reflection-in-response)":[64],"inform":[66],"\"moves\"":[68],"towards":[69],"a":[70,102],"desired":[71],"result;":[72],"(3)":[74],"Grounding":[75],"instantiate":[77],"an":[78,81],"instrument":[79],"from":[80,87,115,129],"example,":[82],"result,":[83],"extrapolation":[85],"directly":[86],"another":[88],"instrument.":[89],"Further,":[90],"leverage":[92],"LLM's":[93],"suggest,":[95],"vary,":[96],"new":[99],"instruments,":[100],"enabling":[101],"system":[103],"that":[104],"goes":[105],"beyond":[106],"hard-coded":[107],"functionality":[108],"by":[109],"generating":[110],"its":[111],"own":[112],"instrumental":[113],"controls":[114],"content.":[116],"We":[117],"demonstrate":[118],"four":[119],"technology":[120],"probes,":[121],"applied":[122],"image":[124],"generation,":[125],"qualitative":[127],"insights":[128],"twelve":[130],"participants,":[131],"showing":[132],"how":[133],"address":[135],"challenges":[136],"intent":[138],"formulation,":[139],"steering":[140],"direct":[142],"manipulation,":[143],"non-linear":[145],"iterative":[146],"workflows":[147],"reflect":[149],"resolve":[151],"intents.":[153]},"counts_by_year":[{"year":2026,"cited_by_count":8},{"year":2025,"cited_by_count":6}],"updated_date":"2026-05-07T13:39:58.223016","created_date":"2025-04-25T00:00:00"}