{"id":"https://openalex.org/W4385152092","doi":"https://doi.org/10.1109/taslp.2023.3297958","title":"Modelling Inter-Rater Uncertainty in Spoken Language Assessment","display_name":"Modelling Inter-Rater Uncertainty in Spoken Language Assessment","publication_year":2023,"publication_date":"2023-01-01","ids":{"openalex":"https://openalex.org/W4385152092","doi":"https://doi.org/10.1109/taslp.2023.3297958"},"language":"en","primary_location":{"id":"doi:10.1109/taslp.2023.3297958","is_oa":false,"landing_page_url":"http://dx.doi.org/10.1109/taslp.2023.3297958","pdf_url":null,"source":{"id":"https://openalex.org/S4210169297","display_name":"IEEE/ACM Transactions on Audio Speech and Language Processing","issn_l":"2329-9290","issn":["2329-9290","2329-9304"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE/ACM Transactions on Audio, Speech, and Language Processing","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5048201022","display_name":"Jeremy H. M. Wong","orcid":"https://orcid.org/0000-0003-3742-7510"},"institutions":[{"id":"https://openalex.org/I115228651","display_name":"Agency for Science, Technology and Research","ror":"https://ror.org/036wvzt09","country_code":"SG","type":"government","lineage":["https://openalex.org/I115228651"]},{"id":"https://openalex.org/I3005327000","display_name":"Institute for Infocomm Research","ror":"https://ror.org/053rfa017","country_code":"SG","type":"facility","lineage":["https://openalex.org/I115228651","https://openalex.org/I3005327000","https://openalex.org/I91275662"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Jeremy H. M. Wong","raw_affiliation_strings":["Institute for Infocomm Research, A*STAR, Singapore"],"raw_orcid":"https://orcid.org/0000-0003-3742-7510","affiliations":[{"raw_affiliation_string":"Institute for Infocomm Research, A*STAR, Singapore","institution_ids":["https://openalex.org/I115228651","https://openalex.org/I3005327000"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102856166","display_name":"Huayun Zhang","orcid":null},"institutions":[{"id":"https://openalex.org/I115228651","display_name":"Agency for Science, Technology and Research","ror":"https://ror.org/036wvzt09","country_code":"SG","type":"government","lineage":["https://openalex.org/I115228651"]},{"id":"https://openalex.org/I3005327000","display_name":"Institute for Infocomm Research","ror":"https://ror.org/053rfa017","country_code":"SG","type":"facility","lineage":["https://openalex.org/I115228651","https://openalex.org/I3005327000","https://openalex.org/I91275662"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Huayun Zhang","raw_affiliation_strings":["Institute for Infocomm Research, A*STAR, Singapore"],"raw_orcid":"https://orcid.org/0000-0003-2528-273X","affiliations":[{"raw_affiliation_string":"Institute for Infocomm Research, A*STAR, Singapore","institution_ids":["https://openalex.org/I115228651","https://openalex.org/I3005327000"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5014190404","display_name":"Nancy F. Chen","orcid":"https://orcid.org/0000-0003-0872-5877"},"institutions":[{"id":"https://openalex.org/I115228651","display_name":"Agency for Science, Technology and Research","ror":"https://ror.org/036wvzt09","country_code":"SG","type":"government","lineage":["https://openalex.org/I115228651"]},{"id":"https://openalex.org/I3005327000","display_name":"Institute for Infocomm Research","ror":"https://ror.org/053rfa017","country_code":"SG","type":"facility","lineage":["https://openalex.org/I115228651","https://openalex.org/I3005327000","https://openalex.org/I91275662"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Nancy F. Chen","raw_affiliation_strings":["Institute for Infocomm Research, A*STAR, Singapore"],"raw_orcid":"https://orcid.org/0000-0003-0872-5877","affiliations":[{"raw_affiliation_string":"Institute for Infocomm Research, A*STAR, Singapore","institution_ids":["https://openalex.org/I115228651","https://openalex.org/I3005327000"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.5274,"has_fulltext":false,"cited_by_count":4,"citation_normalized_percentile":{"value":0.71825909,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":97},"biblio":{"volume":"31","issue":null,"first_page":"2886","last_page":"2898"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.9991999864578247,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.9991999864578247,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":0.9958999752998352,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.994700014591217,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6111836433410645},{"id":"https://openalex.org/keywords/scalar","display_name":"Scalar (mathematics)","score":0.5121374130249023},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.48040416836738586},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4684128165245056},{"id":"https://openalex.org/keywords/beta-distribution","display_name":"Beta distribution","score":0.42073068022727966},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.408480703830719},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.3572256565093994},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.3260945677757263},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.3093838691711426},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.22698688507080078}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6111836433410645},{"id":"https://openalex.org/C57691317","wikidata":"https://www.wikidata.org/wiki/Q1289248","display_name":"Scalar (mathematics)","level":2,"score":0.5121374130249023},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.48040416836738586},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4684128165245056},{"id":"https://openalex.org/C21621910","wikidata":"https://www.wikidata.org/wiki/Q756254","display_name":"Beta distribution","level":2,"score":0.42073068022727966},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.408480703830719},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3572256565093994},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3260945677757263},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.3093838691711426},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.22698688507080078},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/taslp.2023.3297958","is_oa":false,"landing_page_url":"http://dx.doi.org/10.1109/taslp.2023.3297958","pdf_url":null,"source":{"id":"https://openalex.org/S4210169297","display_name":"IEEE/ACM Transactions on Audio Speech and Language Processing","issn_l":"2329-9290","issn":["2329-9290","2329-9304"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE/ACM Transactions on Audio, Speech, and Language Processing","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.6600000262260437,"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":67,"referenced_works":["https://openalex.org/W12857174","https://openalex.org/W34303869","https://openalex.org/W144554552","https://openalex.org/W811578723","https://openalex.org/W1494198834","https://openalex.org/W1524333225","https://openalex.org/W1528470941","https://openalex.org/W1531333757","https://openalex.org/W1579853615","https://openalex.org/W1821462560","https://openalex.org/W1904457459","https://openalex.org/W1976526581","https://openalex.org/W1988790447","https://openalex.org/W1991133427","https://openalex.org/W2030026461","https://openalex.org/W2034537249","https://openalex.org/W2064675550","https://openalex.org/W2085628288","https://openalex.org/W2088538739","https://openalex.org/W2091856355","https://openalex.org/W2095705004","https://openalex.org/W2097137621","https://openalex.org/W2105906164","https://openalex.org/W2115055261","https://openalex.org/W2117590177","https://openalex.org/W2117671523","https://openalex.org/W2127374659","https://openalex.org/W2139008940","https://openalex.org/W2144920829","https://openalex.org/W2146334809","https://openalex.org/W2161278885","https://openalex.org/W2164810574","https://openalex.org/W2395106899","https://openalex.org/W2398423749","https://openalex.org/W2402040300","https://openalex.org/W2514741789","https://openalex.org/W2575763737","https://openalex.org/W2600383743","https://openalex.org/W2729190387","https://openalex.org/W2739575494","https://openalex.org/W2749848124","https://openalex.org/W2788907134","https://openalex.org/W2806471870","https://openalex.org/W2908510526","https://openalex.org/W2911546748","https://openalex.org/W2914769449","https://openalex.org/W2948210953","https://openalex.org/W2950170869","https://openalex.org/W2963505832","https://openalex.org/W3015870704","https://openalex.org/W3037150633","https://openalex.org/W3096195729","https://openalex.org/W3096582851","https://openalex.org/W3196891430","https://openalex.org/W3197474936","https://openalex.org/W3197742413","https://openalex.org/W4240665608","https://openalex.org/W4241644338","https://openalex.org/W6605923411","https://openalex.org/W6631362777","https://openalex.org/W6634817459","https://openalex.org/W6638523607","https://openalex.org/W6674330103","https://openalex.org/W6677717300","https://openalex.org/W6735443497","https://openalex.org/W6748865747","https://openalex.org/W6757817989"],"related_works":["https://openalex.org/W2961085424","https://openalex.org/W4224009465","https://openalex.org/W4306674287","https://openalex.org/W4286629047","https://openalex.org/W2071749707","https://openalex.org/W4205958290","https://openalex.org/W4384212932","https://openalex.org/W2548721895","https://openalex.org/W2373456246","https://openalex.org/W4390590544"],"abstract_inverted_index":{"In":[0],"a":[1,70,97,126,147,181,198],"subjective":[2,94],"task,":[3,172],"such":[4],"as":[5,31,42,96,178],"Spoken":[6],"Language":[7],"Assessment":[8],"(SLA),":[9],"the":[10,60,84,102,108,112,118,129,134,140,151,164,167,170,175,210,219],"reference":[11,54,137],"scores":[12,24,55,138],"provided":[13],"by":[14,66,83,111,124,173],"different":[15],"human":[16],"raters":[17,27],"may":[18,73],"vary.":[19],"A":[20],"collection":[21],"of":[22,34,136,180],"annotated":[23],"from":[25,59,139,150,209],"multiple":[26,61,85,113,141],"can":[28],"be":[29],"interpreted":[30],"an":[32,223],"expression":[33],"data":[35],"uncertainty.":[36],"Previous":[37],"studies":[38],"often":[39],"treat":[40],"SLA":[41,171],"classification":[43],"or":[44],"regression":[45],"tasks,":[46],"and":[47,49,122,133,169,189,197,222],"train":[48],"evaluate":[50],"models":[51],"against":[52],"scalar":[53,71,148],"that":[56,80],"were":[57],"computed":[58],"rater":[62,86],"scores,":[63],"for":[64],"example":[65],"majority":[67],"voting.":[68],"However,":[69],"representation":[72],"not":[74],"adequately":[75],"capture":[76,186],"information":[77],"about":[78],"uncertainty":[79,109,188,208],"is":[81,120,195,202],"expressed":[82,110],"scores.":[87],"This":[88,158],"paper":[89,159],"proposes":[90,161],"to":[91,106,145,162,185,204],"reformulate":[92],"this":[93,116],"task":[95],"distribution":[98,135],"fitting":[99],"problem,":[100],"where":[101],"model":[103,119,168,207],"should":[104],"aim":[105],"emulate":[107],"raters.":[114,142],"Toward":[115],"aim,":[117],"trained":[121],"evaluated":[123,217],"computing":[125],"distance":[127],"between":[128,166],"model's":[130,152,176],"output":[131,153,212],"posterior":[132,154],"Different":[143],"methods":[144],"infer":[146],"score":[149,190],"are":[155,216],"also":[156,160],"considered.":[157],"improve":[163],"match":[165],"interpreting":[174],"outputs":[177],"parameters":[179],"beta":[182],"density":[183],"function,":[184],"both":[187],"monotonicity.":[191],"Finally,":[192],"ensemble":[193],"combination":[194,200],"investigated":[196],"novel":[199],"method":[201],"proposed,":[203],"marginalise":[205],"out":[206],"combined":[211],"distribution.":[213],"These":[214],"approaches":[215],"on":[218],"speechocean762":[220],"dataset":[221],"in-house":[224],"Tamil":[225],"dataset.":[226]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":1}],"updated_date":"2026-07-22T07:51:19.307946","created_date":"2025-10-10T00:00:00"}