{"id":"https://openalex.org/W4327910550","doi":"https://doi.org/10.1145/3576840.3578319","title":"Weakly Supervised Turn-level Engagingness Evaluator for Dialogues","display_name":"Weakly Supervised Turn-level Engagingness Evaluator for Dialogues","publication_year":2023,"publication_date":"2023-03-19","ids":{"openalex":"https://openalex.org/W4327910550","doi":"https://doi.org/10.1145/3576840.3578319"},"language":"en","primary_location":{"id":"doi:10.1145/3576840.3578319","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3576840.3578319","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3576840.3578319","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2023 Conference on Human Information Interaction and Retrieval","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3576840.3578319","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5081142628","display_name":"Shaojie Jiang","orcid":"https://orcid.org/0000-0001-6842-6583"},"institutions":[{"id":"https://openalex.org/I887064364","display_name":"University of Amsterdam","ror":"https://ror.org/04dkp9463","country_code":"NL","type":"education","lineage":["https://openalex.org/I887064364"]}],"countries":["NL"],"is_corresponding":true,"raw_author_name":"Shaojie Jiang","raw_affiliation_strings":["University of Amsterdam, Netherlands"],"affiliations":[{"raw_affiliation_string":"University of Amsterdam, Netherlands","institution_ids":["https://openalex.org/I887064364"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5027138711","display_name":"Svitlana Vakulenko","orcid":"https://orcid.org/0000-0002-5278-8886"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Svitlana Vakulenko","raw_affiliation_strings":["Amazon, Spain"],"affiliations":[{"raw_affiliation_string":"Amazon, Spain","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5031439294","display_name":"Maarten de Rijke","orcid":"https://orcid.org/0000-0002-1086-0202"},"institutions":[{"id":"https://openalex.org/I887064364","display_name":"University of Amsterdam","ror":"https://ror.org/04dkp9463","country_code":"NL","type":"education","lineage":["https://openalex.org/I887064364"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Maarten de Rijke","raw_affiliation_strings":["University of Amsterdam, Netherlands"],"affiliations":[{"raw_affiliation_string":"University of Amsterdam, Netherlands","institution_ids":["https://openalex.org/I887064364"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5081142628"],"corresponding_institution_ids":["https://openalex.org/I887064364"],"apc_list":null,"apc_paid":null,"fwci":1.3776,"has_fulltext":true,"cited_by_count":8,"citation_normalized_percentile":{"value":0.84240157,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"258","last_page":"268"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12128","display_name":"AI in Service Interactions","score":0.9940000176429749,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/conversation","display_name":"Conversation","score":0.7547639012336731},{"id":"https://openalex.org/keywords/dialog-box","display_name":"Dialog box","score":0.7431667447090149},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7220805287361145},{"id":"https://openalex.org/keywords/session","display_name":"Session (web analytics)","score":0.7212694883346558},{"id":"https://openalex.org/keywords/metric","display_name":"Metric (unit)","score":0.5945668816566467},{"id":"https://openalex.org/keywords/heuristic","display_name":"Heuristic","score":0.5713731050491333},{"id":"https://openalex.org/keywords/correlation-coefficient","display_name":"Correlation coefficient","score":0.48964840173721313},{"id":"https://openalex.org/keywords/turn-taking","display_name":"Turn-taking","score":0.4760865271091461},{"id":"https://openalex.org/keywords/proxy","display_name":"Proxy (statistics)","score":0.47100797295570374},{"id":"https://openalex.org/keywords/correlation","display_name":"Correlation","score":0.4674457311630249},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4631378650665283},{"id":"https://openalex.org/keywords/domain","display_name":"Domain (mathematical analysis)","score":0.4628514349460602},{"id":"https://openalex.org/keywords/pearson-product-moment-correlation-coefficient","display_name":"Pearson product-moment correlation coefficient","score":0.4314800798892975},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.42364105582237244},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.3819986879825592},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.33232542872428894},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.1402410864830017},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.13408729434013367},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.10912367701530457},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.06403079628944397}],"concepts":[{"id":"https://openalex.org/C2777200299","wikidata":"https://www.wikidata.org/wiki/Q52943","display_name":"Conversation","level":2,"score":0.7547639012336731},{"id":"https://openalex.org/C173853756","wikidata":"https://www.wikidata.org/wiki/Q86915","display_name":"Dialog box","level":2,"score":0.7431667447090149},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7220805287361145},{"id":"https://openalex.org/C2779182362","wikidata":"https://www.wikidata.org/wiki/Q17126187","display_name":"Session (web analytics)","level":2,"score":0.7212694883346558},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.5945668816566467},{"id":"https://openalex.org/C173801870","wikidata":"https://www.wikidata.org/wiki/Q201413","display_name":"Heuristic","level":2,"score":0.5713731050491333},{"id":"https://openalex.org/C2780092901","wikidata":"https://www.wikidata.org/wiki/Q3433612","display_name":"Correlation coefficient","level":2,"score":0.48964840173721313},{"id":"https://openalex.org/C2776352735","wikidata":"https://www.wikidata.org/wiki/Q2313343","display_name":"Turn-taking","level":3,"score":0.4760865271091461},{"id":"https://openalex.org/C2780148112","wikidata":"https://www.wikidata.org/wiki/Q1432581","display_name":"Proxy (statistics)","level":2,"score":0.47100797295570374},{"id":"https://openalex.org/C117220453","wikidata":"https://www.wikidata.org/wiki/Q5172842","display_name":"Correlation","level":2,"score":0.4674457311630249},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4631378650665283},{"id":"https://openalex.org/C36503486","wikidata":"https://www.wikidata.org/wiki/Q11235244","display_name":"Domain (mathematical analysis)","level":2,"score":0.4628514349460602},{"id":"https://openalex.org/C55078378","wikidata":"https://www.wikidata.org/wiki/Q1136628","display_name":"Pearson product-moment correlation coefficient","level":2,"score":0.4314800798892975},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.42364105582237244},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3819986879825592},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.33232542872428894},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.1402410864830017},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.13408729434013367},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.10912367701530457},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.06403079628944397},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0},{"id":"https://openalex.org/C21547014","wikidata":"https://www.wikidata.org/wiki/Q1423657","display_name":"Operations management","level":1,"score":0.0},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3576840.3578319","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3576840.3578319","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3576840.3578319","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2023 Conference on Human Information Interaction and Retrieval","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3576840.3578319","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3576840.3578319","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3576840.3578319","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2023 Conference on Human Information Interaction and Retrieval","raw_type":"proceedings-article"},"sustainable_development_goals":[{"score":0.4000000059604645,"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4"}],"awards":[{"id":"https://openalex.org/G8589651859","display_name":null,"funder_award_id":"China","funder_id":"https://openalex.org/F4320322725","funder_display_name":"China Scholarship Council"}],"funders":[{"id":"https://openalex.org/F4320321800","display_name":"Nederlandse Organisatie voor Wetenschappelijk Onderzoek","ror":"https://ror.org/04jsz6e67"},{"id":"https://openalex.org/F4320322725","display_name":"China Scholarship Council","ror":"https://ror.org/04atp4p48"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4327910550.pdf","grobid_xml":"https://content.openalex.org/works/W4327910550.grobid-xml"},"referenced_works_count":28,"referenced_works":["https://openalex.org/W2526127466","https://openalex.org/W2590822507","https://openalex.org/W2782898451","https://openalex.org/W2838831674","https://openalex.org/W2890140518","https://openalex.org/W2912512851","https://openalex.org/W2913443447","https://openalex.org/W2951583236","https://openalex.org/W2963527228","https://openalex.org/W2963825865","https://openalex.org/W2979702391","https://openalex.org/W2979826702","https://openalex.org/W2990563493","https://openalex.org/W2996044589","https://openalex.org/W3034600233","https://openalex.org/W3034808773","https://openalex.org/W3035656139","https://openalex.org/W3098258760","https://openalex.org/W3114032074","https://openalex.org/W3137075421","https://openalex.org/W3155584966","https://openalex.org/W3159810391","https://openalex.org/W3175622777","https://openalex.org/W3176264844","https://openalex.org/W3197018105","https://openalex.org/W4211072278","https://openalex.org/W4220841150","https://openalex.org/W4225000971"],"related_works":["https://openalex.org/W357196361","https://openalex.org/W3109425891","https://openalex.org/W1036938216","https://openalex.org/W2113714434","https://openalex.org/W2377792686","https://openalex.org/W2027314909","https://openalex.org/W4200439127","https://openalex.org/W829658220","https://openalex.org/W3096637473","https://openalex.org/W2946560178"],"abstract_inverted_index":{"Engagingness":[0,99,119],"is":[1,17,31,48,63],"an":[2,94],"important":[3],"measurement":[4],"for":[5,90,107,115,140],"evaluating":[6,14],"open-domain":[7],"conversational":[8],"systems.":[9],"The":[10,43],"standard":[11],"approach":[12],"to":[13,70,83,131],"dialogue":[15,29,41,62,67,77],"engagingness":[16,86],"by":[18],"measuring":[19,84],"conversation":[20],"turns":[21],"per":[22],"session":[23,57,78],"(CTPS),":[24],"which":[25,102],"implies":[26],"that":[27,49,145],"the":[28,32,36,56,61,76,104,148,153,164],"length":[30],"main":[33,44],"predictor":[34],"of":[35,46,156],"user":[37,73],"engagement":[38,74],"with":[39],"a":[40,66,111,136],"system.":[42],"limitation":[45],"CTPS":[47],"it":[50],"can":[51],"only":[52],"be":[53],"measured":[54],"at":[55],"level,":[58],"i.e.,":[59],"once":[60],"over.":[64],"But":[65],"system":[68],"has":[69],"continuously":[71],"monitor":[72],"throughout":[75],"as":[79,110,135],"well.":[80],"Existing":[81],"approaches":[82],"turn-level":[85],"require":[87,124],"human":[88,125],"annotations":[89,126],"training.":[91],"We":[92,143],"pioneer":[93],"alternative":[95],"approach,":[96],"Weakly":[97,117],"Supervised":[98,118],"Evaluator":[100,120],"(WeSEE),":[101],"uses":[103],"remaining":[105],"depth":[106],"each":[108],"turn":[109],"heuristic":[112],"weak":[113],"label":[114],"engagingness.":[116],"(WeSEE)":[121],"does":[122],"not":[123],"and":[127,163],"also":[128],"relates":[129],"closely":[130],"CTPS,":[132],"thus":[133],"serving":[134],"good":[137],"learning":[138],"proxy":[139],"this":[141],"metric.":[142],"show":[144],"WeSEE":[146],"achieves":[147],"new":[149],"state-of-the-art":[150],"results":[151],"on":[152],"Fine-grained":[154],"Evaluation":[155],"Dialog":[157],"dataset":[158,166],"(0.38":[159],"Spearman":[160,168],"correlation":[161,169],"coefficient)":[162],"DailyDialog":[165],"(0.62":[167],"coefficient).":[170]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":1},{"year":2023,"cited_by_count":3}],"updated_date":"2026-04-14T08:04:32.555800","created_date":"2025-10-10T00:00:00"}