{"id":"https://openalex.org/W4383679928","doi":"https://doi.org/10.1145/3563657.3596138","title":"Herding AI Cats: Lessons from Designing a Chatbot by Prompting GPT-3","display_name":"Herding AI Cats: Lessons from Designing a Chatbot by Prompting GPT-3","alternate_titles":[],"publication_year":2023,"publication_date":"2023-07-09","ids":{"openalex":"https://openalex.org/W4383679928","doi":"https://doi.org/10.1145/3563657.3596138"},"language":"en","primary_location":{"id":"doi:10.1145/3563657.3596138","doi":"https://doi.org/10.1145/3563657.3596138","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3563657.3596138","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3563657.3596138","source":{"id":"https://openalex.org/S4363608268","display_name":"Designing Interactive Systems Conference","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"conference"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2023 ACM Designing Interactive Systems Conference","raw_type":"proceedings-article"},"parent_works":[],"lineage":["https://openalex.org/W4383679928"],"child_works_count":0,"type":"conference-paper","is_traditional":true,"indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3563657.3596138","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5065870025","display_name":"J.D. Zamfirescu-Pereira","orcid":"https://orcid.org/0000-0002-5310-6728","observed_orcids":["https://orcid.org/0000-0002-5310-6728"]},"institutions":[{"id":"https://openalex.org/I95457486","display_name":"University of California, Berkeley","ror":"https://ror.org/01an7q238","country_code":"US","type":"education","lineage":["https://openalex.org/I95457486"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"J.D. Zamfirescu-Pereira","raw_affiliation_strings":["Computer Science, UC Berkeley, United States"],"raw_orcid":"https://orcid.org/0000-0002-5310-6728","affiliations":[{"raw_affiliation_string":"Computer Science, UC Berkeley, United States","institution_ids":["https://openalex.org/I95457486"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Heather Wei","orcid":"https://orcid.org/0009-0006-5429-3170","observed_orcids":["https://orcid.org/0009-0006-5429-3170"]},"institutions":[{"id":"https://openalex.org/I95457486","display_name":"University of California, Berkeley","ror":"https://ror.org/01an7q238","country_code":"US","type":"education","lineage":["https://openalex.org/I95457486"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Heather Wei","raw_affiliation_strings":["UC Berkeley, United States"],"raw_orcid":"https://orcid.org/0009-0006-5429-3170","affiliations":[{"raw_affiliation_string":"UC Berkeley, United States","institution_ids":["https://openalex.org/I95457486"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5092431819","display_name":"Amy Xiao","orcid":"https://orcid.org/0009-0001-5977-0507","observed_orcids":["https://orcid.org/0009-0001-5977-0507"]},"institutions":[{"id":"https://openalex.org/I95457486","display_name":"University of California, Berkeley","ror":"https://ror.org/01an7q238","country_code":"US","type":"education","lineage":["https://openalex.org/I95457486"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Amy Xiao","raw_affiliation_strings":["UC Berkeley, United States"],"raw_orcid":"https://orcid.org/0009-0001-5977-0507","affiliations":[{"raw_affiliation_string":"UC Berkeley, United States","institution_ids":["https://openalex.org/I95457486"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5004716988","display_name":"K Gu","orcid":"https://orcid.org/0009-0008-8095-3704","observed_orcids":["https://orcid.org/0009-0008-8095-3704"]},"institutions":[{"id":"https://openalex.org/I95457486","display_name":"University of California, Berkeley","ror":"https://ror.org/01an7q238","country_code":"US","type":"education","lineage":["https://openalex.org/I95457486"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kitty Gu","raw_affiliation_strings":["UC Berkeley, United States"],"raw_orcid":"https://orcid.org/0009-0008-8095-3704","affiliations":[{"raw_affiliation_string":"UC Berkeley, United States","institution_ids":["https://openalex.org/I95457486"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5108302121","display_name":"Grace Jung","orcid":"https://orcid.org/0000-0002-0863-7415","observed_orcids":["https://orcid.org/0000-0002-0863-7415"]},"institutions":[{"id":"https://openalex.org/I95457486","display_name":"University of California, Berkeley","ror":"https://ror.org/01an7q238","country_code":"US","type":"education","lineage":["https://openalex.org/I95457486"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Grace Jung","raw_affiliation_strings":["UC Berkeley, United States"],"raw_orcid":"https://orcid.org/0000-0002-0863-7415","affiliations":[{"raw_affiliation_string":"UC Berkeley, United States","institution_ids":["https://openalex.org/I95457486"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Matthew G Lee","orcid":"https://orcid.org/0000-0002-0863-7415","observed_orcids":["https://orcid.org/0000-0002-0863-7415"]},"institutions":[{"id":"https://openalex.org/I95457486","display_name":"University of California, Berkeley","ror":"https://ror.org/01an7q238","country_code":"US","type":"education","lineage":["https://openalex.org/I95457486"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Matthew G Lee","raw_affiliation_strings":["University of California, Berkeley, United States"],"raw_orcid":"https://orcid.org/0000-0002-0863-7415","affiliations":[{"raw_affiliation_string":"University of California, Berkeley, United States","institution_ids":["https://openalex.org/I95457486"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5084965768","display_name":"Bj\u00f6rn Hartmann","orcid":"https://orcid.org/0000-0002-0693-0829","observed_orcids":["https://orcid.org/0000-0002-0693-0829"]},"institutions":[{"id":"https://openalex.org/I95457486","display_name":"University of California, Berkeley","ror":"https://ror.org/01an7q238","country_code":"US","type":"education","lineage":["https://openalex.org/I95457486"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Bjoern Hartmann","raw_affiliation_strings":["EECS, UC Berkeley, United States"],"raw_orcid":"https://orcid.org/0000-0002-0693-0829","affiliations":[{"raw_affiliation_string":"EECS, UC Berkeley, United States","institution_ids":["https://openalex.org/I95457486"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5017881649","display_name":"Qian Yang","orcid":"https://orcid.org/0000-0002-3548-2535","observed_orcids":["https://orcid.org/0000-0002-3548-2535"]},"institutions":[{"id":"https://openalex.org/I205783295","display_name":"Cornell University","ror":"https://ror.org/05bnh6r87","country_code":"US","type":"education","lineage":["https://openalex.org/I205783295"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Qian Yang","raw_affiliation_strings":["Computing and Information Science, Cornell University, United States"],"raw_orcid":"https://orcid.org/0000-0002-3548-2535","affiliations":[{"raw_affiliation_string":"Computing and Information Science, Cornell University, United States","institution_ids":["https://openalex.org/I205783295"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":38.1394,"has_fulltext":true,"cited_by_count":103,"citation_normalized_percentile":{"value":0.999332,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":97,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"2206","last_page":"2220"},"is_retracted":false,"is_pulled":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12128","display_name":"AI in Service Interactions","score":0.5532000064849854,"subfield":{"id":"https://openalex.org/subfields/1406","display_name":"Marketing"},"field":{"id":"https://openalex.org/fields/14","display_name":"Business, Management and Accounting"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T12128","display_name":"AI in Service Interactions","score":0.5532000064849854,"subfield":{"id":"https://openalex.org/subfields/1406","display_name":"Marketing"},"field":{"id":"https://openalex.org/fields/14","display_name":"Business, Management and Accounting"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.2304999977350235,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10803","display_name":"Innovative Human-Technology Interaction","score":0.09600000083446503,"subfield":{"id":"https://openalex.org/subfields/1709","display_name":"Human-Computer Interaction"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/chatbot-design","display_name":"chatbot design","score":0.7979999780654907},{"id":"https://openalex.org/keywords/gpt-3","display_name":"GPT-3","score":0.9860000014305115},{"id":"https://openalex.org/keywords/gpt-4","display_name":"GPT-4","score":0.9300000071525574},{"id":"https://openalex.org/keywords/user-experience","display_name":"user experience","score":0.8130000233650208},{"id":"https://openalex.org/keywords/prompt-engineering","display_name":"prompt engineering","score":0.6460000276565552},{"id":"https://openalex.org/keywords/guardrails","display_name":"guardrails","score":0.5540000200271606}],"concepts":[{"id":"https://openalex.org/C177605951","wikidata":"https://www.wikidata.org/wiki/Q1484503","display_name":"Herding","level":2,"score":0.930992603302002},{"id":"https://openalex.org/C2779041454","wikidata":"https://www.wikidata.org/wiki/Q870780","display_name":"Chatbot","level":2,"score":0.9212228059768677},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6380665302276611},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.5259188413619995},{"id":"https://openalex.org/C201025465","wikidata":"https://www.wikidata.org/wiki/Q11248500","display_name":"User experience design","level":2,"score":0.4222789406776428},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.2657511234283447},{"id":"https://openalex.org/C95457728","wikidata":"https://www.wikidata.org/wiki/Q309","display_name":"History","level":0,"score":0.09437689185142517},{"id":"https://openalex.org/C166957645","wikidata":"https://www.wikidata.org/wiki/Q23498","display_name":"Archaeology","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3563657.3596138","doi":"https://doi.org/10.1145/3563657.3596138","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3563657.3596138","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3563657.3596138","source":{"id":"https://openalex.org/S4363608268","display_name":"Designing Interactive Systems Conference","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"conference"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2023 ACM Designing Interactive Systems Conference","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3563657.3596138","doi":"https://doi.org/10.1145/3563657.3596138","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3563657.3596138","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3563657.3596138","source":{"id":"https://openalex.org/S4363608268","display_name":"Designing Interactive Systems Conference","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"conference"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2023 ACM Designing Interactive Systems Conference","raw_type":"proceedings-article"},"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G3134685295","display_name":null,"funder_award_id":"FA8750-20-C0156","funder_id":"https://openalex.org/F4320332180","funder_display_name":"Defense Advanced Research Projects Agency"},{"id":"https://openalex.org/G4713059963","display_name":null,"funder_award_id":"FA8750","funder_id":"https://openalex.org/F4320332180","funder_display_name":"Defense Advanced Research Projects Agency"},{"id":"https://openalex.org/G7087593930","display_name":null,"funder_award_id":"FA8750-20-C-0156, FA8750-20-C-0074, and FA8750-20-C0155","funder_id":"https://openalex.org/F4320332180","funder_display_name":"Defense Advanced Research Projects Agency"}],"funders":[{"id":"https://openalex.org/F4320332180","display_name":"Defense Advanced Research Projects Agency","ror":"https://ror.org/02caytj08"},{"id":"https://openalex.org/F4320332467","display_name":"U.S. Air Force","ror":"https://ror.org/006gmme17"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4383679928.pdf","grobid_xml":"https://content.openalex.org/works/W4383679928.grobid-xml"},"referenced_works_count":25,"referenced_works":["https://openalex.org/W2080000436","https://openalex.org/W2091909921","https://openalex.org/W2104123546","https://openalex.org/W2139470221","https://openalex.org/W2171036849","https://openalex.org/W2808450727","https://openalex.org/W2933377349","https://openalex.org/W2952607215","https://openalex.org/W3003223899","https://openalex.org/W3030163527","https://openalex.org/W3032875465","https://openalex.org/W3135634497","https://openalex.org/W3185341429","https://openalex.org/W3203321135","https://openalex.org/W3205068155","https://openalex.org/W3217422184","https://openalex.org/W4220747294","https://openalex.org/W4221159404","https://openalex.org/W4221159410","https://openalex.org/W4318931874","https://openalex.org/W4319653533","https://openalex.org/W4361289889","https://openalex.org/W4366548330","https://openalex.org/W6600127060","https://openalex.org/W6736880212"],"related_works":["https://openalex.org/W7038481105","https://openalex.org/W4366548330","https://openalex.org/W4297166494","https://openalex.org/W4396219010","https://openalex.org/W4319653533","https://openalex.org/W7202002141","https://openalex.org/W4321162035","https://openalex.org/W4413081225","https://openalex.org/W7169864057","https://openalex.org/W7163519571"],"abstract_inverted_index":{"Prompting":[0],"Large":[1],"Language":[2],"Models":[3],"(LLMs)":[4],"is":[5],"an":[6],"exciting":[7],"new":[8],"approach":[9],"to":[10,23,29,80,114,122,131],"designing":[11],"chatbots.":[12,136],"But":[13],"can":[14],"it":[15,100],"improve":[16],"LLM\u2019s":[17],"user":[18],"experience":[19],"(UX)":[20],"reliably":[21,94],"enough":[22],"power":[24],"chatbot":[25,33],"products?":[26],"Our":[27],"attempt":[28],"design":[30,72,123,151,158],"a":[31,75,109],"robust":[32],"by":[34],"prompting":[35],"GPT-3/4":[36],"alone":[37],"suggests:":[38],"not":[39,49,64],"yet.":[40],"Prompts":[41],"made":[42],"achieving":[43],"\u201c80%\u201d":[44],"UX":[45,67,115,150],"goals":[46],"easy,":[47],"but":[48],"the":[50,54,102,132],"remaining":[51,56],"20%.":[52],"Fixing":[53],"few":[55],"interaction":[57],"breakdowns":[58],"resembled":[59],"herding":[60],"cats:":[61],"We":[62],"could":[63,91],"address":[65],"one":[66,71],"issue":[68],"or":[69],"test":[70],"solution":[73],"at":[74,85],"time;":[76],"instead,":[77],"we":[78],"had":[79,105],"handle":[81],"everything":[82],"everywhere":[83],"all":[84],"once.":[86],"Moreover,":[87],"because":[88],"no":[89,106],"prompt":[90],"make":[92],"GPT":[93],"say":[95],"\u201cI":[96],"don\u2019t":[97],"know\u201d":[98],"when":[99],"should,":[101],"user-GPT":[103],"conversations":[104],"guardrails":[107],"after":[108],"breakdown":[110],"occurred,":[111],"often":[112],"leading":[113],"downward":[116],"spirals.":[117],"These":[118],"risks":[119],"incentivized":[120],"us":[121],"highly":[124],"prescriptive":[125],"prompts":[126],"and":[127,146,153,160],"scripted":[128],"bots,":[129],"counter":[130],"promises":[133],"of":[134],"LLM-powered":[135],"This":[137],"paper":[138],"describes":[139],"this":[140],"case":[141],"study,":[142],"unpacks":[143],"prompting\u2019s":[144],"fickleness":[145],"its":[147],"impact":[148],"on":[149],"processes,":[152],"discusses":[154],"implications":[155],"for":[156],"LLM-based":[157],"methods":[159],"tools.":[161]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W4383679928","counts_by_year":[{"year":2026,"cited_by_count":26},{"year":2025,"cited_by_count":44},{"year":2024,"cited_by_count":27},{"year":2023,"cited_by_count":6}],"updated_date":"2026-11-19T14:11:59.513116","created_date":"2023-07-10T00:00:00"}
