{"id":"https://openalex.org/W2970820898","doi":"https://doi.org/10.21437/interspeech.2019-3033","title":"Benchmarking Benchmarks: Introducing New Automatic Indicators for Benchmarking Spoken Language Understanding Corpora","display_name":"Benchmarking Benchmarks: Introducing New Automatic Indicators for Benchmarking Spoken Language Understanding Corpora","publication_year":2019,"publication_date":"2019-09-13","ids":{"openalex":"https://openalex.org/W2970820898","doi":"https://doi.org/10.21437/interspeech.2019-3033","mag":"2970820898"},"language":"en","primary_location":{"id":"doi:10.21437/interspeech.2019-3033","is_oa":false,"landing_page_url":"https://doi.org/10.21437/interspeech.2019-3033","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Interspeech 2019","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Fr\u00e9d\u00e9ric B\u00e9chet","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Fr\u00e9d\u00e9ric B\u00e9chet","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5073795682","display_name":"Christian Raymond","orcid":"https://orcid.org/0000-0002-0963-9367"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Christian Raymond","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":8,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.9987999796867371,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmarking","display_name":"Benchmarking","score":0.9791231155395508},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.783284604549408},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.5431025624275208},{"id":"https://openalex.org/keywords/spoken-language","display_name":"Spoken language","score":0.5329647064208984},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.43406838178634644},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3957844376564026},{"id":"https://openalex.org/keywords/geography","display_name":"Geography","score":0.046104997396469116}],"concepts":[{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.9791231155395508},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.783284604549408},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.5431025624275208},{"id":"https://openalex.org/C2776230583","wikidata":"https://www.wikidata.org/wiki/Q1322198","display_name":"Spoken language","level":2,"score":0.5329647064208984},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.43406838178634644},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3957844376564026},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.046104997396469116},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.0},{"id":"https://openalex.org/C162853370","wikidata":"https://www.wikidata.org/wiki/Q39809","display_name":"Marketing","level":1,"score":0.0},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.21437/interspeech.2019-3033","is_oa":false,"landing_page_url":"https://doi.org/10.21437/interspeech.2019-3033","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Interspeech 2019","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","score":0.8100000023841858,"display_name":"Quality Education"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":6,"referenced_works":["https://openalex.org/W1904365287","https://openalex.org/W2157331557","https://openalex.org/W2160986985","https://openalex.org/W2166293310","https://openalex.org/W2803609229","https://openalex.org/W2889201969"],"related_works":["https://openalex.org/W4238897586","https://openalex.org/W435179959","https://openalex.org/W2619091065","https://openalex.org/W2341842940","https://openalex.org/W2059640416","https://openalex.org/W1490753184","https://openalex.org/W2284465472","https://openalex.org/W2770764537","https://openalex.org/W3192589309","https://openalex.org/W2159086674"],"abstract_inverted_index":{"Empirical":[0],"evaluation":[1,6],"is":[2,36,147,155],"nowadays":[3],"the":[4,14,38,96,100,107,125,185],"main":[5],"paradigm":[7],"in":[8,50,77,95,117],"Natural":[9],"Language":[10,46],"Processing":[11],"for":[12,27,40,123,192],"assessing":[13,124,184],"relevance":[15,126,186],"of":[16,127,167,189],"a":[17,55,120,165],"new":[18,121],"machine-learning":[19],"based":[20,59],"model.":[21],"If":[22],"large":[23],"corpora":[24,174],"are":[25,69,81],"available":[26],"tasks":[28,42],"such":[29,43,64],"as":[30,44,65,103],"Automatic":[31],"Speech":[32],"Recognition":[33],",":[34,88],"this":[35,118],"not":[37,85,141],"case":[39],"other":[41],"Spoken":[45],"Understanding":[47],"(SLU),":[48],"consisting":[49],"translating":[51],"spoken":[52],"transcriptions":[53],"into":[54,136],"formal":[56],"representation":[57],"often":[58,82],"on":[60,106,164],"semantic":[61],"frames.":[62],"Corpora":[63],"ATIS":[66,108,111],"or":[67,99],"SNIPS":[68],"widely":[70],"used":[71],"to":[72,157],"compare":[73],"systems,":[74],"however":[75],"differences":[76],"performance":[78,139],"among":[79],"systems":[80,138,170],"very":[83],"small,":[84],"statistically":[86],"significant":[87],"and":[89,153,171,179],"can":[90],"be":[91,158],"produced":[92],"by":[93,149],"biases":[94],"data":[97],"collection":[98],"annotation":[101],"scheme,":[102],"we":[104],"presented":[105],"corpus":[109,191],"(Is":[110],"too":[112],"shallow?,":[113],"IS2018).":[114],"We":[115,131,160],"propose":[116],"study":[119],"methodology":[122,163],"an":[128],"SLU":[129,169,194],"corpus.":[130],"claim":[132],"that":[133],"only":[134],"taking":[135],"account":[137],"does":[140],"provide":[142],"enough":[143],"insight":[144],"about":[145],"what":[146,154],"covered":[148],"current":[150],"state-of-the-art":[151],"models":[152],"left":[156],"done.":[159],"apply":[161],"our":[162],"set":[166],"4":[168],"5":[172],"benchmark":[173],"(ATIS,":[175],"SNIPS,":[176],"M2M,":[177],"MEDIA)":[178],"automatically":[180],"produce":[181],"several":[182],"indicators":[183],"(or":[187],"not)":[188],"each":[190],"benchmarking":[193],"models.":[195]},"counts_by_year":[{"year":2024,"cited_by_count":1},{"year":2023,"cited_by_count":2},{"year":2022,"cited_by_count":1},{"year":2021,"cited_by_count":2},{"year":2020,"cited_by_count":2}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}