{"id":"https://openalex.org/W3200765157","doi":"https://doi.org/10.1017/s1351324923000372","title":"Actionable conversational quality indicators for improving task-oriented dialog systems","display_name":"Actionable conversational quality indicators for improving task-oriented dialog systems","publication_year":2024,"publication_date":"2024-01-09","ids":{"openalex":"https://openalex.org/W3200765157","doi":"https://doi.org/10.1017/s1351324923000372","mag":"3200765157"},"language":"en","primary_location":{"id":"doi:10.1017/s1351324923000372","is_oa":true,"landing_page_url":"https://doi.org/10.1017/s1351324923000372","pdf_url":"https://www.cambridge.org/core/services/aop-cambridge-core/content/view/97B0827A8F75093990BCF487469225BC/S1351324923000372a.pdf/div-class-title-actionable-conversational-quality-indicators-for-improving-task-oriented-dialog-systems-div.pdf","source":{"id":"https://openalex.org/S18088403","display_name":"Natural Language Engineering","issn_l":"1351-3249","issn":["1351-3249","1469-8110"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310311721","host_organization_name":"Cambridge University Press","host_organization_lineage":["https://openalex.org/P4310311721","https://openalex.org/P4310311702"],"host_organization_lineage_names":["Cambridge University Press","University of Cambridge"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Natural Language Engineering","raw_type":"journal-article"},"type":"article","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://www.cambridge.org/core/services/aop-cambridge-core/content/view/97B0827A8F75093990BCF487469225BC/S1351324923000372a.pdf/div-class-title-actionable-conversational-quality-indicators-for-improving-task-oriented-dialog-systems-div.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5067735440","display_name":"Michael A. Higgins","orcid":"https://orcid.org/0000-0003-1665-9101"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Michael Higgins","raw_affiliation_strings":["LivePerson Inc. New York, NY, USA"],"affiliations":[{"raw_affiliation_string":"LivePerson Inc. New York, NY, USA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5051773175","display_name":"Dominic Widdows","orcid":"https://orcid.org/0000-0002-4241-0820"},"institutions":[{"id":"https://openalex.org/I4210119158","display_name":"IonQ (United States)","ror":"https://ror.org/02ft1jb97","country_code":"US","type":"company","lineage":["https://openalex.org/I4210119158"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Dominic Widdows","raw_affiliation_strings":["IonQ Inc. College Park, MD, USA","LivePerson Inc. New York, NY, USA"],"affiliations":[{"raw_affiliation_string":"IonQ Inc. College Park, MD, USA","institution_ids":["https://openalex.org/I4210119158"]},{"raw_affiliation_string":"LivePerson Inc. New York, NY, USA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5010808154","display_name":"Beth Ann Hockey","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Beth Ann Hockey","raw_affiliation_strings":["LivePerson Inc. New York, NY, USA"],"affiliations":[{"raw_affiliation_string":"LivePerson Inc. New York, NY, USA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5042575746","display_name":"Akshay Hazare","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Akshay Hazare","raw_affiliation_strings":["LivePerson Inc. New York, NY, USA"],"affiliations":[{"raw_affiliation_string":"LivePerson Inc. New York, NY, USA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5039387733","display_name":"Kristen Howell","orcid":"https://orcid.org/0000-0001-8468-884X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kristen Howell","raw_affiliation_strings":["LivePerson Inc. New York, NY, USA"],"affiliations":[{"raw_affiliation_string":"LivePerson Inc. New York, NY, USA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5079660356","display_name":"Gwen Christian","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gwen Christian","raw_affiliation_strings":["LivePerson Inc. New York, NY, USA"],"affiliations":[{"raw_affiliation_string":"LivePerson Inc. New York, NY, USA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5057439011","display_name":"Sujit Mathi","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sujit Mathi","raw_affiliation_strings":["LivePerson Inc. New York, NY, USA"],"affiliations":[{"raw_affiliation_string":"LivePerson Inc. New York, NY, USA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5109351552","display_name":"C. Brew","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chris Brew","raw_affiliation_strings":["LivePerson Inc. New York, NY, USA"],"affiliations":[{"raw_affiliation_string":"LivePerson Inc. New York, NY, USA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5036008088","display_name":"Andrew P. Maurer","orcid":"https://orcid.org/0000-0002-5470-237X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Andrew Maurer","raw_affiliation_strings":["LivePerson Inc. New York, NY, USA"],"affiliations":[{"raw_affiliation_string":"LivePerson Inc. New York, NY, USA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5055442034","display_name":"George Bonev","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"George Bonev","raw_affiliation_strings":["LivePerson Inc. New York, NY, USA"],"affiliations":[{"raw_affiliation_string":"LivePerson Inc. New York, NY, USA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5104000416","display_name":"Matthew Dunn","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Matthew Dunn","raw_affiliation_strings":["LivePerson Inc. New York, NY, USA"],"affiliations":[{"raw_affiliation_string":"LivePerson Inc. New York, NY, USA","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5045317776","display_name":"Joseph Bradley","orcid":"https://orcid.org/0000-0002-6430-1094"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Joseph Bradley","raw_affiliation_strings":["LivePerson Inc. New York, NY, USA"],"affiliations":[{"raw_affiliation_string":"LivePerson Inc. New York, NY, USA","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":12,"corresponding_author_ids":["https://openalex.org/A5051773175"],"corresponding_institution_ids":["https://openalex.org/I4210119158"],"apc_list":null,"apc_paid":null,"fwci":0.7273,"has_fulltext":true,"cited_by_count":2,"citation_normalized_percentile":{"value":0.72698939,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":95,"max":96},"biblio":{"volume":"30","issue":"6","first_page":"1229","last_page":"1254"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12128","display_name":"AI in Service Interactions","score":0.9984999895095825,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/dialog-box","display_name":"Dialog box","score":0.8977373838424683},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8394361138343811},{"id":"https://openalex.org/keywords/dialog-system","display_name":"Dialog system","score":0.6769148707389832},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.6741411685943604},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.6094249486923218},{"id":"https://openalex.org/keywords/conversation","display_name":"Conversation","score":0.5784716606140137},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.5134575963020325},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.5115348100662231},{"id":"https://openalex.org/keywords/categorization","display_name":"Categorization","score":0.49060145020484924},{"id":"https://openalex.org/keywords/service","display_name":"Service (business)","score":0.48981186747550964},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4185810983181},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.26977139711380005}],"concepts":[{"id":"https://openalex.org/C173853756","wikidata":"https://www.wikidata.org/wiki/Q86915","display_name":"Dialog box","level":2,"score":0.8977373838424683},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8394361138343811},{"id":"https://openalex.org/C190954187","wikidata":"https://www.wikidata.org/wiki/Q5270587","display_name":"Dialog system","level":3,"score":0.6769148707389832},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.6741411685943604},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.6094249486923218},{"id":"https://openalex.org/C2777200299","wikidata":"https://www.wikidata.org/wiki/Q52943","display_name":"Conversation","level":2,"score":0.5784716606140137},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.5134575963020325},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.5115348100662231},{"id":"https://openalex.org/C94124525","wikidata":"https://www.wikidata.org/wiki/Q912550","display_name":"Categorization","level":2,"score":0.49060145020484924},{"id":"https://openalex.org/C2780378061","wikidata":"https://www.wikidata.org/wiki/Q25351891","display_name":"Service (business)","level":2,"score":0.48981186747550964},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4185810983181},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.26977139711380005},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C136264566","wikidata":"https://www.wikidata.org/wiki/Q159810","display_name":"Economy","level":1,"score":0.0},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1017/s1351324923000372","is_oa":true,"landing_page_url":"https://doi.org/10.1017/s1351324923000372","pdf_url":"https://www.cambridge.org/core/services/aop-cambridge-core/content/view/97B0827A8F75093990BCF487469225BC/S1351324923000372a.pdf/div-class-title-actionable-conversational-quality-indicators-for-improving-task-oriented-dialog-systems-div.pdf","source":{"id":"https://openalex.org/S18088403","display_name":"Natural Language Engineering","issn_l":"1351-3249","issn":["1351-3249","1469-8110"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310311721","host_organization_name":"Cambridge University Press","host_organization_lineage":["https://openalex.org/P4310311721","https://openalex.org/P4310311702"],"host_organization_lineage_names":["Cambridge University Press","University of Cambridge"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Natural Language Engineering","raw_type":"journal-article"},{"id":"pmh:oai:arXiv.org:2109.11064","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2109.11064","pdf_url":"https://arxiv.org/pdf/2109.11064","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"}],"best_oa_location":{"id":"doi:10.1017/s1351324923000372","is_oa":true,"landing_page_url":"https://doi.org/10.1017/s1351324923000372","pdf_url":"https://www.cambridge.org/core/services/aop-cambridge-core/content/view/97B0827A8F75093990BCF487469225BC/S1351324923000372a.pdf/div-class-title-actionable-conversational-quality-indicators-for-improving-task-oriented-dialog-systems-div.pdf","source":{"id":"https://openalex.org/S18088403","display_name":"Natural Language Engineering","issn_l":"1351-3249","issn":["1351-3249","1469-8110"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310311721","host_organization_name":"Cambridge University Press","host_organization_lineage":["https://openalex.org/P4310311721","https://openalex.org/P4310311702"],"host_organization_lineage_names":["Cambridge University Press","University of Cambridge"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Natural Language Engineering","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3200765157.pdf"},"referenced_works_count":45,"referenced_works":["https://openalex.org/W10329047","https://openalex.org/W79013291","https://openalex.org/W178897730","https://openalex.org/W969718702","https://openalex.org/W1508665521","https://openalex.org/W1604513301","https://openalex.org/W1622397659","https://openalex.org/W2004224263","https://openalex.org/W2005434739","https://openalex.org/W2021284652","https://openalex.org/W2035444264","https://openalex.org/W2078098467","https://openalex.org/W2102770351","https://openalex.org/W2151814822","https://openalex.org/W2154290668","https://openalex.org/W2164777277","https://openalex.org/W2174767898","https://openalex.org/W2251175910","https://openalex.org/W2337241406","https://openalex.org/W2610185579","https://openalex.org/W2775878099","https://openalex.org/W2799176105","https://openalex.org/W2808216081","https://openalex.org/W2886943638","https://openalex.org/W2888093771","https://openalex.org/W2891103209","https://openalex.org/W2896457183","https://openalex.org/W2949769095","https://openalex.org/W2949782788","https://openalex.org/W2962711864","https://openalex.org/W2963301433","https://openalex.org/W2970303069","https://openalex.org/W2970641574","https://openalex.org/W2980037344","https://openalex.org/W2990629495","https://openalex.org/W2997094605","https://openalex.org/W3000779003","https://openalex.org/W3004248456","https://openalex.org/W3035572844","https://openalex.org/W3035629539","https://openalex.org/W3088291589","https://openalex.org/W3104405162","https://openalex.org/W3104901442","https://openalex.org/W4247250990","https://openalex.org/W6786130232"],"related_works":["https://openalex.org/W2500779211","https://openalex.org/W48079147","https://openalex.org/W326836678","https://openalex.org/W1963944933","https://openalex.org/W2563921006","https://openalex.org/W1600043506","https://openalex.org/W2111550420","https://openalex.org/W2735573723","https://openalex.org/W3133893348","https://openalex.org/W2549666521"],"abstract_inverted_index":{"Abstract":[0],"Automatic":[1],"dialog":[2,27,86,134,148,162],"systems":[3,15,28,163],"have":[4,130,139],"become":[5,39],"a":[6,206,211,221,235,243,253],"mainstream":[7],"part":[8],"of":[9,62,95,108,124,128,145,156,185,215,229,249,266],"online":[10],"customer":[11,22,167],"service.":[12],"Many":[13],"such":[14,242],"are":[16,102,191,201],"built,":[17],"maintained,":[18],"and":[19,30,37,52,68,91,114,170,183,219],"improved":[20,113],"by":[21,263],"service":[23,168],"specialists,":[24],"rather":[25,82],"than":[26,83],"engineers":[29],"computer":[31],"programmers.":[32],"As":[33],"conversations":[34],"between":[35],"people":[36],"machines":[38],"commonplace,":[40],"it":[41],"is":[42,47,50,150],"critical":[43],"to":[44,58,71,105,115,118,141,193,204],"understand":[45],"what":[46,49,53],"working,":[48],"not,":[51],"actions":[54,252],"can":[55,111],"be":[56,72,112,261],"taken":[57],"reduce":[59],"the":[60,79,84,93,143,147,154,172,181,216,227,231,247],"frequency":[61],"inappropriate":[63],"system":[64,149],"responses.":[65],"These":[66],"analyses":[67],"recommendations":[69],"need":[70],"presented":[73],"in":[74,165,195],"terms":[75],"that":[76,110,240],"directly":[77],"reflect":[78],"user":[80],"experience":[81],"internal":[85,161],"processing.":[87],"This":[88,121],"paper":[89],"introduces":[90],"explains":[92],"use":[94],"Actionable":[96],"Conversational":[97],"Quality":[98],"Indicators":[99],"(ACQIs),":[100],"which":[101,129,189,209],"used":[103,164,203],"both":[104],"recognize":[106],"parts":[107],"dialogs":[109],"recommend":[116],"how":[117],"improve":[119],"them.":[120],"combines":[122],"benefits":[123],"previous":[125],"approaches,":[126],"some":[127],"focused":[131],"on":[132,159,171,180],"producing":[133],"quality":[135],"scoring":[136],"while":[137],"others":[138],"sought":[140],"categorize":[142],"types":[144],"errors":[146],"making.":[151],"We":[152,178,238],"demonstrate":[153],"effectiveness":[155],"using":[157],"ACQIs":[158,190],"LivePerson":[160],"commercial":[166],"applications":[169],"publicly":[173],"available":[174],"LEGOv2":[175],"conversational":[176,186],"dataset.":[177],"report":[179],"annotation":[182],"analysis":[184],"datasets":[187,200],"showing":[188],"important":[192],"fix":[194],"various":[196],"situations.":[197],"The":[198],"annotated":[199],"then":[202],"build":[205],"predictive":[207],"model":[208,244],"uses":[210],"turn-based":[212],"vector":[213],"embedding":[214],"message":[217],"texts":[218],"achieves":[220],"79%":[222],"weighted":[223],"average":[224,265],"f1-measure":[225],"at":[226,257],"task":[228],"finding":[230],"correct":[232],"ACQI":[233],"for":[234],"given":[236],"conversation.":[237],"predict":[239],"if":[241],"worked":[245],"perfectly,":[246],"range":[248],"potential":[250],"improvement":[251],"bot-builder":[254],"must":[255],"consider":[256],"each":[258],"turn":[259],"could":[260],"reduced":[262],"an":[264],"81%.":[267]},"counts_by_year":[{"year":2025,"cited_by_count":2}],"updated_date":"2026-03-10T16:38:18.471706","created_date":"2025-10-10T00:00:00"}