{"id":"https://openalex.org/W4400611399","doi":"https://doi.org/10.1145/3652988.3673967","title":"On LLM Wizards: Identifying Large Language Models' Behaviors for Wizard of Oz Experiments","display_name":"On LLM Wizards: Identifying Large Language Models' Behaviors for Wizard of Oz Experiments","publication_year":2024,"publication_date":"2024-09-16","ids":{"openalex":"https://openalex.org/W4400611399","doi":"https://doi.org/10.1145/3652988.3673967"},"language":"en","primary_location":{"id":"doi:10.1145/3652988.3673967","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3652988.3673967","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM International Conference on Intelligent Virtual Agents","raw_type":"proceedings-article"},"type":"preprint","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2407.08067","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5038772455","display_name":"Jingchao Fang","orcid":"https://orcid.org/0000-0002-9412-4244"},"institutions":[{"id":"https://openalex.org/I4391768151","display_name":"Toyota Research Institute","ror":"https://ror.org/04fpkc108","country_code":null,"type":"facility","lineage":["https://openalex.org/I4210125472","https://openalex.org/I4391768151"]},{"id":"https://openalex.org/I917207718","display_name":"Toyota Industries (United States)","ror":"https://ror.org/0132ebr60","country_code":"US","type":"company","lineage":["https://openalex.org/I4210125472","https://openalex.org/I4210145315","https://openalex.org/I917207718"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Jingchao Fang","raw_affiliation_strings":["Toyota Research Institute, USA","Toyota Research Institute Los Altos, California, USA","Research , California, USA"],"affiliations":[{"raw_affiliation_string":"Toyota Research Institute, USA","institution_ids":["https://openalex.org/I917207718","https://openalex.org/I4391768151"]},{"raw_affiliation_string":"Toyota Research Institute Los Altos, California, USA","institution_ids":["https://openalex.org/I4391768151"]},{"raw_affiliation_string":"Research , California, USA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5068636555","display_name":"Nikos Ar\u00e9chiga","orcid":"https://orcid.org/0009-0005-5585-7006"},"institutions":[{"id":"https://openalex.org/I4391768151","display_name":"Toyota Research Institute","ror":"https://ror.org/04fpkc108","country_code":null,"type":"facility","lineage":["https://openalex.org/I4210125472","https://openalex.org/I4391768151"]},{"id":"https://openalex.org/I917207718","display_name":"Toyota Industries (United States)","ror":"https://ror.org/0132ebr60","country_code":"US","type":"company","lineage":["https://openalex.org/I4210125472","https://openalex.org/I4210145315","https://openalex.org/I917207718"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Nikos Arechiga","raw_affiliation_strings":["Toyota Research Institute, USA","Research , California, USA","Toyota Research Institute Los Altos, California, USA"],"affiliations":[{"raw_affiliation_string":"Toyota Research Institute, USA","institution_ids":["https://openalex.org/I917207718","https://openalex.org/I4391768151"]},{"raw_affiliation_string":"Research , California, USA","institution_ids":[]},{"raw_affiliation_string":"Toyota Research Institute Los Altos, California, USA","institution_ids":["https://openalex.org/I4391768151"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5063558614","display_name":"Keiichi Namikoshi","orcid":null},"institutions":[{"id":"https://openalex.org/I4391768151","display_name":"Toyota Research Institute","ror":"https://ror.org/04fpkc108","country_code":null,"type":"facility","lineage":["https://openalex.org/I4210125472","https://openalex.org/I4391768151"]},{"id":"https://openalex.org/I917207718","display_name":"Toyota Industries (United States)","ror":"https://ror.org/0132ebr60","country_code":"US","type":"company","lineage":["https://openalex.org/I4210125472","https://openalex.org/I4210145315","https://openalex.org/I917207718"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Keiichi Namikoshi","raw_affiliation_strings":["Toyota Research Institute, USA"],"affiliations":[{"raw_affiliation_string":"Toyota Research Institute, USA","institution_ids":["https://openalex.org/I917207718","https://openalex.org/I4391768151"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5045951556","display_name":"Nayeli Suseth Bravo","orcid":"https://orcid.org/0000-0001-9238-9831"},"institutions":[{"id":"https://openalex.org/I4391768151","display_name":"Toyota Research Institute","ror":"https://ror.org/04fpkc108","country_code":null,"type":"facility","lineage":["https://openalex.org/I4210125472","https://openalex.org/I4391768151"]},{"id":"https://openalex.org/I917207718","display_name":"Toyota Industries (United States)","ror":"https://ror.org/0132ebr60","country_code":"US","type":"company","lineage":["https://openalex.org/I4210125472","https://openalex.org/I4210145315","https://openalex.org/I917207718"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Nayeli Bravo","raw_affiliation_strings":["Toyota Research Institute, USA","Toyota Research Institute Los Altos, California, USA","Research , California, USA"],"affiliations":[{"raw_affiliation_string":"Toyota Research Institute, USA","institution_ids":["https://openalex.org/I917207718","https://openalex.org/I4391768151"]},{"raw_affiliation_string":"Toyota Research Institute Los Altos, California, USA","institution_ids":["https://openalex.org/I4391768151"]},{"raw_affiliation_string":"Research , California, USA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5091314417","display_name":"Candice Hogan","orcid":"https://orcid.org/0000-0002-3240-2560"},"institutions":[{"id":"https://openalex.org/I4391768151","display_name":"Toyota Research Institute","ror":"https://ror.org/04fpkc108","country_code":null,"type":"facility","lineage":["https://openalex.org/I4210125472","https://openalex.org/I4391768151"]},{"id":"https://openalex.org/I917207718","display_name":"Toyota Industries (United States)","ror":"https://ror.org/0132ebr60","country_code":"US","type":"company","lineage":["https://openalex.org/I4210125472","https://openalex.org/I4210145315","https://openalex.org/I917207718"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Candice Hogan","raw_affiliation_strings":["Toyota Research Institute, USA","Research , California, USA","Toyota Research Institute Los Altos, California, USA"],"affiliations":[{"raw_affiliation_string":"Toyota Research Institute, USA","institution_ids":["https://openalex.org/I917207718","https://openalex.org/I4391768151"]},{"raw_affiliation_string":"Research , California, USA","institution_ids":[]},{"raw_affiliation_string":"Toyota Research Institute Los Altos, California, USA","institution_ids":["https://openalex.org/I4391768151"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5001270504","display_name":"David A. Shamma","orcid":"https://orcid.org/0000-0003-2399-9374"},"institutions":[{"id":"https://openalex.org/I4391768151","display_name":"Toyota Research Institute","ror":"https://ror.org/04fpkc108","country_code":null,"type":"facility","lineage":["https://openalex.org/I4210125472","https://openalex.org/I4391768151"]},{"id":"https://openalex.org/I917207718","display_name":"Toyota Industries (United States)","ror":"https://ror.org/0132ebr60","country_code":"US","type":"company","lineage":["https://openalex.org/I4210125472","https://openalex.org/I4210145315","https://openalex.org/I917207718"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"David A. Shamma","raw_affiliation_strings":["Toyota Research Institute, USA"],"affiliations":[{"raw_affiliation_string":"Toyota Research Institute, USA","institution_ids":["https://openalex.org/I917207718","https://openalex.org/I4391768151"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5038772455"],"corresponding_institution_ids":["https://openalex.org/I4391768151","https://openalex.org/I917207718"],"apc_list":null,"apc_paid":null,"fwci":3.0427,"has_fulltext":true,"cited_by_count":9,"citation_normalized_percentile":{"value":0.92285366,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"11"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10664","display_name":"Sentiment Analysis and Opinion Mining","score":0.9965999722480774,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10664","display_name":"Sentiment Analysis and Opinion Mining","score":0.9965999722480774,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9959999918937683,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12128","display_name":"AI in Service Interactions","score":0.9901999831199646,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/wizard","display_name":"Wizard","score":0.7842270135879517},{"id":"https://openalex.org/keywords/wizard-of-oz","display_name":"Wizard of oz","score":0.5640637874603271},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.49881744384765625},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.2604392468929291},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.26020747423171997}],"concepts":[{"id":"https://openalex.org/C2776784076","wikidata":"https://www.wikidata.org/wiki/Q580334","display_name":"Wizard","level":2,"score":0.7842270135879517},{"id":"https://openalex.org/C2987423310","wikidata":"https://www.wikidata.org/wiki/Q18159725","display_name":"Wizard of oz","level":2,"score":0.5640637874603271},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.49881744384765625},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.2604392468929291},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.26020747423171997}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3652988.3673967","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3652988.3673967","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM International Conference on Intelligent Virtual Agents","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2407.08067","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2407.08067","pdf_url":"https://arxiv.org/pdf/2407.08067","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2407.08067","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2407.08067","pdf_url":"https://arxiv.org/pdf/2407.08067","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4400611399.pdf","grobid_xml":"https://content.openalex.org/works/W4400611399.grobid-xml"},"referenced_works_count":59,"referenced_works":["https://openalex.org/W1597864774","https://openalex.org/W1969152782","https://openalex.org/W1972051027","https://openalex.org/W1980231763","https://openalex.org/W2036263953","https://openalex.org/W2087609837","https://openalex.org/W2092795326","https://openalex.org/W2099813784","https://openalex.org/W2106336379","https://openalex.org/W2115310639","https://openalex.org/W2127135847","https://openalex.org/W2157392897","https://openalex.org/W2251455484","https://openalex.org/W2550419310","https://openalex.org/W2611049140","https://openalex.org/W2739819123","https://openalex.org/W2755801862","https://openalex.org/W2787096751","https://openalex.org/W2793427265","https://openalex.org/W2915786288","https://openalex.org/W2940530202","https://openalex.org/W2942662004","https://openalex.org/W2945397504","https://openalex.org/W3000144571","https://openalex.org/W3015322406","https://openalex.org/W3029922759","https://openalex.org/W3034224415","https://openalex.org/W3081789885","https://openalex.org/W3090433062","https://openalex.org/W3100355250","https://openalex.org/W3102474786","https://openalex.org/W3104331765","https://openalex.org/W3104546989","https://openalex.org/W3123340107","https://openalex.org/W3135734416","https://openalex.org/W3153370697","https://openalex.org/W3160112384","https://openalex.org/W3160643684","https://openalex.org/W3185652159","https://openalex.org/W4200131949","https://openalex.org/W4221143046","https://openalex.org/W4288080268","https://openalex.org/W4319165762","https://openalex.org/W4321455981","https://openalex.org/W4360806508","https://openalex.org/W4365999098","https://openalex.org/W4366549000","https://openalex.org/W4366549203","https://openalex.org/W4366591012","https://openalex.org/W4366599154","https://openalex.org/W4376117416","https://openalex.org/W4385066501","https://openalex.org/W4385571232","https://openalex.org/W4386200967","https://openalex.org/W4388497846","https://openalex.org/W4389519254","https://openalex.org/W4389519898","https://openalex.org/W4389524193","https://openalex.org/W6778883912"],"related_works":["https://openalex.org/W4246078203","https://openalex.org/W659328528","https://openalex.org/W1550648368","https://openalex.org/W2127135847","https://openalex.org/W4402443486","https://openalex.org/W603474643","https://openalex.org/W2152818631","https://openalex.org/W1545061903","https://openalex.org/W1504394863","https://openalex.org/W2757160751"],"abstract_inverted_index":{"The":[0],"Wizard":[1,15],"of":[2,80,133],"Oz":[3],"(WoZ)":[4],"method":[5],"is":[6],"a":[7,13,17,77,125],"widely":[8],"adopted":[9],"research":[10],"approach":[11],"where":[12],"human":[14],"\u201crole-plays\u201d":[16],"not":[18],"readily":[19],"available":[20],"technology":[21],"and":[22,30,59,76,110,140],"interacts":[23],"with":[24,56],"participants":[25],"to":[26,45,103],"elicit":[27],"user":[28],"behaviors":[29],"probe":[31],"the":[32,36,63,93,131],"design":[33],"space.":[34],"With":[35],"growing":[37],"ability":[38,83,136],"for":[39,101],"modern":[40],"large":[41],"language":[42],"models":[43],"(LLMs)":[44],"role-play,":[46],"one":[47],"can":[48],"apply":[49],"LLMs":[50,72,106],"as":[51],"Wizards":[52,118],"in":[53,73,137],"WoZ":[54,74,89,108,138],"experiments":[55,75,109,139],"better":[57],"scalability":[58],"lower":[60],"cost":[61],"than":[62],"traditional":[64],"approach.":[65],"However,":[66],"methodological":[67],"guidance":[68],"on":[69],"responsibly":[70],"applying":[71],"systematic":[78],"evaluation":[79,127],"LLMs\u2019":[81,134,142],"role-playing":[82,135],"are":[84],"lacking.":[85],"Through":[86],"two":[87],"LLM-powered":[88],"studies,":[90],"we":[91],"take":[92],"first":[94],"step":[95],"towards":[96],"identifying":[97],"an":[98],"experiment":[99],"lifecycle":[100],"researchers":[102],"safely":[104],"integrate":[105],"into":[107],"interpret":[111],"data":[112],"generated":[113],"from":[114],"settings":[115],"that":[116,129],"involve":[117],"role-played":[119],"by":[120],"LLMs.":[121],"We":[122],"also":[123],"contribute":[124],"heuristic-based":[126],"framework":[128],"allows":[130],"estimation":[132],"reveals":[141],"behavior":[143],"patterns":[144],"at":[145],"scale.":[146]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":7}],"updated_date":"2026-04-14T08:04:32.555800","created_date":"2024-07-14T00:00:00"}