{"id":"https://openalex.org/W4400397487","doi":"https://doi.org/10.1145/3640794.3665889","title":"Our Dialogue System Sucks - but Luckily we are at the Top of the Leaderboard!: A Discussion on Current Practices in NLP Evaluation","display_name":"Our Dialogue System Sucks - but Luckily we are at the Top of the Leaderboard!: A Discussion on Current Practices in NLP Evaluation","publication_year":2024,"publication_date":"2024-07-07","ids":{"openalex":"https://openalex.org/W4400397487","doi":"https://doi.org/10.1145/3640794.3665889"},"language":"en","primary_location":{"id":"doi:10.1145/3640794.3665889","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3640794.3665889","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Conversational User Interfaces 2024","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://research.tilburguniversity.edu/en/publications/9501bb04-410b-475d-b3d8-94e3a2fadf20","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5016616705","display_name":"Anouck Braggaar","orcid":"https://orcid.org/0000-0002-7284-0442"},"institutions":[{"id":"https://openalex.org/I193700539","display_name":"Tilburg University","ror":"https://ror.org/04b8v1s79","country_code":"NL","type":"education","lineage":["https://openalex.org/I193700539"]}],"countries":["NL"],"is_corresponding":true,"raw_author_name":"Anouck Braggaar","raw_affiliation_strings":["Department of Communication and Cognition, Tilburg School of Humanities and Digital Sciences, Tilburg University, Netherlands"],"raw_orcid":"https://orcid.org/0000-0002-7284-0442","affiliations":[{"raw_affiliation_string":"Department of Communication and Cognition, Tilburg School of Humanities and Digital Sciences, Tilburg University, Netherlands","institution_ids":["https://openalex.org/I193700539"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5068617833","display_name":"Linwei He","orcid":"https://orcid.org/0000-0002-6593-1661"},"institutions":[{"id":"https://openalex.org/I193700539","display_name":"Tilburg University","ror":"https://ror.org/04b8v1s79","country_code":"NL","type":"education","lineage":["https://openalex.org/I193700539"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Linwei He","raw_affiliation_strings":["Department of Communication and Cognition, Tilburg School of Humanities and Digital Sciences, Tilburg University, Netherlands"],"raw_orcid":"https://orcid.org/0000-0002-6593-1661","affiliations":[{"raw_affiliation_string":"Department of Communication and Cognition, Tilburg School of Humanities and Digital Sciences, Tilburg University, Netherlands","institution_ids":["https://openalex.org/I193700539"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5046673724","display_name":"Jan de Wit","orcid":"https://orcid.org/0000-0002-7299-7992"},"institutions":[{"id":"https://openalex.org/I193700539","display_name":"Tilburg University","ror":"https://ror.org/04b8v1s79","country_code":"NL","type":"education","lineage":["https://openalex.org/I193700539"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Jan De Wit","raw_affiliation_strings":["Department of Communication and Cognition, Tilburg School of Humanities and Digital Sciences, Tilburg University, Netherlands"],"raw_orcid":"https://orcid.org/0000-0002-7299-7992","affiliations":[{"raw_affiliation_string":"Department of Communication and Cognition, Tilburg School of Humanities and Digital Sciences, Tilburg University, Netherlands","institution_ids":["https://openalex.org/I193700539"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5016616705"],"corresponding_institution_ids":["https://openalex.org/I193700539"],"apc_list":null,"apc_paid":null,"fwci":1.0668,"has_fulltext":false,"cited_by_count":3,"citation_normalized_percentile":{"value":0.77323986,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":91,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"5"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.9965000152587891,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.9965000152587891,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12128","display_name":"AI in Service Interactions","score":0.9959999918937683,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10456","display_name":"Multi-Agent Systems and Negotiation","score":0.9909999966621399,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6560493111610413},{"id":"https://openalex.org/keywords/current","display_name":"Current (fluid)","score":0.5571495890617371},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.4752245545387268},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.411773681640625},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.3695468008518219},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.3444942831993103},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.24143734574317932},{"id":"https://openalex.org/keywords/electrical-engineering","display_name":"Electrical engineering","score":0.16870763897895813}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6560493111610413},{"id":"https://openalex.org/C148043351","wikidata":"https://www.wikidata.org/wiki/Q4456944","display_name":"Current (fluid)","level":2,"score":0.5571495890617371},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.4752245545387268},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.411773681640625},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.3695468008518219},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3444942831993103},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.24143734574317932},{"id":"https://openalex.org/C119599485","wikidata":"https://www.wikidata.org/wiki/Q43035","display_name":"Electrical engineering","level":1,"score":0.16870763897895813}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1145/3640794.3665889","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3640794.3665889","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Conversational User Interfaces 2024","raw_type":"proceedings-article"},{"id":"pmh:oai:tilburguniversity.edu:openaire_cris_publications/9501bb04-410b-475d-b3d8-94e3a2fadf20","is_oa":true,"landing_page_url":"https://research.tilburguniversity.edu/en/publications/9501bb04-410b-475d-b3d8-94e3a2fadf20","pdf_url":null,"source":{"id":"https://openalex.org/S4306401490","display_name":"Research portal (Tilburg University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I193700539","host_organization_name":"Tilburg University","host_organization_lineage":["https://openalex.org/I193700539"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Braggaar, A, He, L & Wit, J D 2024, Our Dialogue System Sucks - but Luckily we are at the Top of the Leaderboard!: A Discussion on Current Practices in NLP Evaluation. in Proceedings of the 6th Conference on ACM Conversational User Interfaces, CUI 2024., 48, Proceedings of the 6th Conference on ACM Conversational User Interfaces, CUI 2024. https://doi.org/10.1145/3640794.3665889","raw_type":"info:eu-repo/semantics/publishedVersion"},{"id":"pmh:oai:tilburguniversity.edu:publications/9501bb04-410b-475d-b3d8-94e3a2fadf20","is_oa":true,"landing_page_url":"https://repository.tilburguniversity.edu/bitstreams/b0a2f96d-e8d7-49c6-9b26-a339b623623b/download","pdf_url":null,"source":{"id":"https://openalex.org/S4306401490","display_name":"Research portal (Tilburg University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I193700539","host_organization_name":"Tilburg University","host_organization_lineage":["https://openalex.org/I193700539"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Braggaar, A, He, L & Wit, J D 2024, Our Dialogue System Sucks - but Luckily we are at the Top of the Leaderboard!: A Discussion on Current Practices in NLP Evaluation. in Proceedings of the 6th Conference on ACM Conversational User Interfaces, CUI 2024., 48, Proceedings of the 6th Conference on ACM Conversational User Interfaces, CUI 2024. https://doi.org/10.1145/3640794.3665889","raw_type":"info:eu-repo/semantics/publishedVersion"}],"best_oa_location":{"id":"pmh:oai:tilburguniversity.edu:openaire_cris_publications/9501bb04-410b-475d-b3d8-94e3a2fadf20","is_oa":true,"landing_page_url":"https://research.tilburguniversity.edu/en/publications/9501bb04-410b-475d-b3d8-94e3a2fadf20","pdf_url":null,"source":{"id":"https://openalex.org/S4306401490","display_name":"Research portal (Tilburg University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I193700539","host_organization_name":"Tilburg University","host_organization_lineage":["https://openalex.org/I193700539"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Braggaar, A, He, L & Wit, J D 2024, Our Dialogue System Sucks - but Luckily we are at the Top of the Leaderboard!: A Discussion on Current Practices in NLP Evaluation. in Proceedings of the 6th Conference on ACM Conversational User Interfaces, CUI 2024., 48, Proceedings of the 6th Conference on ACM Conversational User Interfaces, CUI 2024. https://doi.org/10.1145/3640794.3665889","raw_type":"info:eu-repo/semantics/publishedVersion"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G1164390919","display_name":"Smooth Operator. Development and effects of personalized conversational AI.","funder_award_id":"KIVI.2019.009","funder_id":"https://openalex.org/F4320321800","funder_display_name":"Nederlandse Organisatie voor Wetenschappelijk Onderzoek"}],"funders":[{"id":"https://openalex.org/F4320321800","display_name":"Nederlandse Organisatie voor Wetenschappelijk Onderzoek","ror":"https://ror.org/04jsz6e67"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":17,"referenced_works":["https://openalex.org/W1990364369","https://openalex.org/W2003453784","https://openalex.org/W2033458507","https://openalex.org/W2883231496","https://openalex.org/W3088691004","https://openalex.org/W3107855336","https://openalex.org/W3163791097","https://openalex.org/W3205211246","https://openalex.org/W4210647952","https://openalex.org/W4221039286","https://openalex.org/W4313908157","https://openalex.org/W4321610465","https://openalex.org/W4384520759","https://openalex.org/W4389518911","https://openalex.org/W4390348173","https://openalex.org/W4391576639","https://openalex.org/W4403175460"],"related_works":["https://openalex.org/W1982412832","https://openalex.org/W4244464241","https://openalex.org/W2384573129","https://openalex.org/W2351224547","https://openalex.org/W2358945257","https://openalex.org/W4318692582","https://openalex.org/W4368618351","https://openalex.org/W2006863447","https://openalex.org/W2299124854","https://openalex.org/W3204019825"],"abstract_inverted_index":{"Currently,":[0],"leaderboards":[1,29],"are":[2],"often":[3],"used":[4],"to":[5,80],"evaluate":[6],"natural":[7],"language":[8,16],"processing":[9],"(NLP)":[10],"systems":[11],"and":[12,30,69],"in":[13,37,42,55,74],"particular":[14],"large":[15],"models.":[17,44],"In":[18],"this":[19],"paper":[20],"we":[21,24],"argue":[22],"why":[23],"should":[25,49,64],"step":[26],"away":[27],"from":[28],"follow":[31],"a":[32],"more":[33],"inclusive":[34,67],"approach":[35,68],"both":[36],"developing":[38],"as":[39,41],"well":[40],"evaluating":[43],"The":[45],"focus":[46],"of":[47,72],"evaluation":[48],"be":[50],"on":[51],"the":[52,57],"complete":[53],"context":[54],"which":[56],"system":[58],"operates.":[59],"To":[60],"accomplish":[61],"this,":[62],"researchers":[63],"take":[65,70],"an":[66],"note":[71],"developments":[73],"multiple":[75],"scientific":[76],"fields":[77],"(from":[78],"NLP":[79],"communication":[81],"science).":[82]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":1}],"updated_date":"2026-07-29T14:22:42.915294","created_date":"2025-10-10T00:00:00"}