{"id":"https://openalex.org/W2739575494","doi":"https://doi.org/10.18653/v1/p17-2008","title":"Incorporating Uncertainty into Deep Learning for Spoken Language Assessment","display_name":"Incorporating Uncertainty into Deep Learning for Spoken Language Assessment","publication_year":2017,"publication_date":"2017-01-01","ids":{"openalex":"https://openalex.org/W2739575494","doi":"https://doi.org/10.18653/v1/p17-2008","mag":"2739575494"},"language":"en","primary_location":{"id":"doi:10.18653/v1/p17-2008","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/p17-2008","pdf_url":"https://www.aclweb.org/anthology/P17-2008.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 55th Annual Meeting of the Association for\n          Computational Linguistics (Volume 2: Short Papers)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://www.aclweb.org/anthology/P17-2008.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5080001860","display_name":"Andrey Malinin","orcid":null},"institutions":[{"id":"https://openalex.org/I241749","display_name":"University of Cambridge","ror":"https://ror.org/013meh722","country_code":"GB","type":"education","lineage":["https://openalex.org/I241749"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Andrey Malinin","raw_affiliation_strings":["University of Cambridge, Department of Engineering Trumpington St, Cambridge CB2 1PZ, UK"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Cambridge, Department of Engineering Trumpington St, Cambridge CB2 1PZ, UK","institution_ids":["https://openalex.org/I241749"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5024283093","display_name":"Anton Ragni","orcid":"https://orcid.org/0000-0003-0634-4456"},"institutions":[{"id":"https://openalex.org/I241749","display_name":"University of Cambridge","ror":"https://ror.org/013meh722","country_code":"GB","type":"education","lineage":["https://openalex.org/I241749"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Anton Ragni","raw_affiliation_strings":["University of Cambridge, Department of Engineering Trumpington St, Cambridge CB2 1PZ, UK"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Cambridge, Department of Engineering Trumpington St, Cambridge CB2 1PZ, UK","institution_ids":["https://openalex.org/I241749"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5111076409","display_name":"Kate Knill","orcid":null},"institutions":[{"id":"https://openalex.org/I241749","display_name":"University of Cambridge","ror":"https://ror.org/013meh722","country_code":"GB","type":"education","lineage":["https://openalex.org/I241749"]}],"countries":["GB"],"is_corresponding":true,"raw_author_name":"Kate Knill","raw_affiliation_strings":["University of Cambridge, Department of Engineering Trumpington St, Cambridge CB2 1PZ, UK"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Cambridge, Department of Engineering Trumpington St, Cambridge CB2 1PZ, UK","institution_ids":["https://openalex.org/I241749"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5050766679","display_name":"Mark Gales","orcid":"https://orcid.org/0000-0002-5311-8219"},"institutions":[{"id":"https://openalex.org/I241749","display_name":"University of Cambridge","ror":"https://ror.org/013meh722","country_code":"GB","type":"education","lineage":["https://openalex.org/I241749"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Mark Gales","raw_affiliation_strings":["University of Cambridge, Department of Engineering Trumpington St, Cambridge CB2 1PZ, UK"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Cambridge, Department of Engineering Trumpington St, Cambridge CB2 1PZ, UK","institution_ids":["https://openalex.org/I241749"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5111076409"],"corresponding_institution_ids":["https://openalex.org/I241749"],"apc_list":null,"apc_paid":null,"fwci":2.7022,"has_fulltext":true,"cited_by_count":29,"citation_normalized_percentile":{"value":0.92350748,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"45","last_page":"50"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12814","display_name":"Gaussian Processes and Bayesian Inference","score":0.9977999925613403,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12814","display_name":"Gaussian Processes and Bayesian Inference","score":0.9977999925613403,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10320","display_name":"Neural Networks and Applications","score":0.9812999963760376,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T14470","display_name":"Advanced Data Processing Techniques","score":0.9778000116348267,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7650104761123657},{"id":"https://openalex.org/keywords/dropout","display_name":"Dropout (neural networks)","score":0.5924052596092224},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5374430418014526},{"id":"https://openalex.org/keywords/gaussian-process","display_name":"Gaussian process","score":0.5225683450698853},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.48121026158332825},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.47256842255592346},{"id":"https://openalex.org/keywords/deep-neural-networks","display_name":"Deep neural networks","score":0.46520811319351196},{"id":"https://openalex.org/keywords/monte-carlo-method","display_name":"Monte Carlo method","score":0.43802163004875183},{"id":"https://openalex.org/keywords/spoken-language","display_name":"Spoken language","score":0.43217337131500244},{"id":"https://openalex.org/keywords/deep-learning","display_name":"Deep learning","score":0.4226526618003845},{"id":"https://openalex.org/keywords/gaussian","display_name":"Gaussian","score":0.346666157245636},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.1269637942314148}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7650104761123657},{"id":"https://openalex.org/C2776145597","wikidata":"https://www.wikidata.org/wiki/Q25339462","display_name":"Dropout (neural networks)","level":2,"score":0.5924052596092224},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5374430418014526},{"id":"https://openalex.org/C61326573","wikidata":"https://www.wikidata.org/wiki/Q1496376","display_name":"Gaussian process","level":3,"score":0.5225683450698853},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.48121026158332825},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.47256842255592346},{"id":"https://openalex.org/C2984842247","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep neural networks","level":3,"score":0.46520811319351196},{"id":"https://openalex.org/C19499675","wikidata":"https://www.wikidata.org/wiki/Q232207","display_name":"Monte Carlo method","level":2,"score":0.43802163004875183},{"id":"https://openalex.org/C2776230583","wikidata":"https://www.wikidata.org/wiki/Q1322198","display_name":"Spoken language","level":2,"score":0.43217337131500244},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.4226526618003845},{"id":"https://openalex.org/C163716315","wikidata":"https://www.wikidata.org/wiki/Q901177","display_name":"Gaussian","level":2,"score":0.346666157245636},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.1269637942314148},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0}],"mesh":[],"locations_count":5,"locations":[{"id":"doi:10.18653/v1/p17-2008","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/p17-2008","pdf_url":"https://www.aclweb.org/anthology/P17-2008.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 55th Annual Meeting of the Association for\n          Computational Linguistics (Volume 2: Short Papers)","raw_type":"proceedings-article"},{"id":"pmh:oai:eprints.whiterose.ac.uk:152824","is_oa":false,"landing_page_url":"http://eprints.whiterose.ac.uk/152824/","pdf_url":null,"source":{"id":"https://openalex.org/S4306400854","display_name":"White Rose Research Online (University of Leeds, The University of Sheffield, University of York)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I2800616092","host_organization_name":"White Rose University Consortium","host_organization_lineage":["https://openalex.org/I2800616092"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Proceedings Paper"},{"id":"pmh:oai:generic.eprints.org:912757","is_oa":false,"landing_page_url":"http://publications.eng.cam.ac.uk/912757/","pdf_url":null,"source":{"id":"https://openalex.org/S4406922847","display_name":"Cambridge University Engineering Department Publications Database","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Conference or Workshop Item"},{"id":"pmh:oai:www.repository.cam.ac.uk:1810/279180","is_oa":false,"landing_page_url":"https://www.repository.cam.ac.uk/handle/1810/279180","pdf_url":null,"source":{"id":"https://openalex.org/S4306401777","display_name":"Apollo (University of Cambridge)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I241749","host_organization_name":"University of Cambridge","host_organization_lineage":["https://openalex.org/I241749"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Conference Object"},{"id":"doi:10.17863/cam.26560","is_oa":true,"landing_page_url":"https://doi.org/10.17863/cam.26560","pdf_url":null,"source":{"id":"https://openalex.org/S7407050737","display_name":"Apollo","issn_l":null,"issn":[],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"doi:10.18653/v1/p17-2008","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/p17-2008","pdf_url":"https://www.aclweb.org/anthology/P17-2008.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 55th Annual Meeting of the Association for\n          Computational Linguistics (Volume 2: Short Papers)","raw_type":"proceedings-article"},"sustainable_development_goals":[{"score":0.7699999809265137,"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education"}],"awards":[],"funders":[{"id":"https://openalex.org/F4320334627","display_name":"Engineering and Physical Sciences Research Council","ror":"https://ror.org/0439y7842"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2739575494.pdf","grobid_xml":"https://content.openalex.org/works/W2739575494.grobid-xml"},"referenced_works_count":17,"referenced_works":["https://openalex.org/W582134693","https://openalex.org/W1522301498","https://openalex.org/W1533861849","https://openalex.org/W1579853615","https://openalex.org/W1746819321","https://openalex.org/W1959608418","https://openalex.org/W1979364015","https://openalex.org/W2030026461","https://openalex.org/W2095705004","https://openalex.org/W2101234009","https://openalex.org/W2161988551","https://openalex.org/W2251058486","https://openalex.org/W2304127486","https://openalex.org/W2575763737","https://openalex.org/W2964059111","https://openalex.org/W2964121744","https://openalex.org/W4211049957"],"related_works":["https://openalex.org/W3082178636","https://openalex.org/W1521968289","https://openalex.org/W2782041652","https://openalex.org/W2952088488","https://openalex.org/W2392157706","https://openalex.org/W2599192953","https://openalex.org/W2169866437","https://openalex.org/W1964286703","https://openalex.org/W4287183600","https://openalex.org/W4321151651"],"abstract_inverted_index":{"There":[0],"is":[1,61,175],"a":[2,41,75,124,144],"growing":[3],"demand":[4],"for":[5,63],"automatic":[6],"assessment":[7],"of":[8,27,52,77],"spoken":[9],"English":[10],"proficiency.":[11],"These":[12],"systems":[13,65],"need":[14],"to":[15,23,44,68,73,85,114,127,133,146,177],"handle":[16],"large":[17,103],"variations":[18],"in":[19,80,184],"input":[20],"data":[21,47,104,152,164],"owing":[22],"the":[24,45,50,53,166,172],"wide":[25],"range":[26],"candidate":[28],"skill":[29],"levels":[30],"and":[31,33,130,135,153,180],"L1s,":[32],"errors":[34],"from":[35,165],"ASR.":[36],"Some":[37],"candidates":[38],"will":[39],"be":[40,112],"poor":[42],"match":[43],"training":[46,151],"set,":[48],"undermining":[49],"validity":[51],"predicted":[54],"grade.":[55],"For":[56],"high":[57,154],"stakes":[58],"tests":[59],"it":[60,132],"essential":[62],"such":[64],"not":[66,99],"only":[67],"grade":[69],"well,":[70],"but":[71],"also":[72,111],"provide":[74,115],"measure":[76],"their":[78,81],"uncertainty":[79,116,129,149,155],"predictions,":[82],"enabling":[83],"rejection":[84,186],"human":[86],"graders.":[87],"Previous":[88],"work":[89],"examined":[90],"Gaussian":[91],"Process":[92],"(GP)":[93],"graders":[94],"which,":[95],"though":[96],"successful,":[97],"do":[98],"scale":[100],"well":[101],"with":[102,137,182],"sets.":[105],"Deep":[106],"Neural":[107],"Networks":[108],"(DNN)":[109],"may":[110],"used":[113],"using":[117],"Monte-Carlo":[118],"Dropout":[119],"(MCD).":[120],"This":[121],"paper":[122],"proposes":[123],"novel":[125],"method":[126],"yield":[128],"compares":[131],"GPs":[134,179],"DNNs":[136,181],"MCD.":[138],"The":[139],"proposed":[140,173],"approach":[141,174],"explicitly":[142],"teaches":[143],"DNN":[145],"have":[147],"low":[148],"on":[150,156,163],"generated":[157],"artificial":[158],"data.":[159],"On":[160],"experiments":[161],"conducted":[162],"Business":[167],"Language":[168],"Testing":[169],"Service":[170],"(BULATS),":[171],"found":[176],"outperform":[178],"MCD":[183],"uncertainty-based":[185],"whilst":[187],"achieving":[188],"comparable":[189],"grading":[190],"performance.":[191]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":1},{"year":2023,"cited_by_count":4},{"year":2021,"cited_by_count":7},{"year":2020,"cited_by_count":5},{"year":2019,"cited_by_count":3},{"year":2018,"cited_by_count":4},{"year":2017,"cited_by_count":1}],"updated_date":"2026-04-28T14:05:53.105641","created_date":"2025-10-10T00:00:00"}