{"id":"https://openalex.org/W7148585322","doi":"https://doi.org/10.1109/asru65441.2025.11434693","title":"Beyond Modality Limitations: A Unified MLLM Approach to Automated Speaking Assessment with Effective Curriculum Learning","display_name":"Beyond Modality Limitations: A Unified MLLM Approach to Automated Speaking Assessment with Effective Curriculum Learning","publication_year":2025,"publication_date":"2025-12-06","ids":{"openalex":"https://openalex.org/W7148585322","doi":"https://doi.org/10.1109/asru65441.2025.11434693"},"language":null,"primary_location":{"id":"doi:10.1109/asru65441.2025.11434693","is_oa":false,"landing_page_url":"https://doi.org/10.1109/asru65441.2025.11434693","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 IEEE Automatic Speech Recognition and Understanding Workshop (ASRU)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5132802095","display_name":"Yu-Hsuan Fang","orcid":null},"institutions":[{"id":"https://openalex.org/I134161618","display_name":"National Taiwan Normal University","ror":"https://ror.org/059dkdx38","country_code":"TW","type":"education","lineage":["https://openalex.org/I134161618"]}],"countries":["TW"],"is_corresponding":true,"raw_author_name":"Yu-Hsuan Fang","raw_affiliation_strings":["National Taiwan Normal University"],"affiliations":[{"raw_affiliation_string":"National Taiwan Normal University","institution_ids":["https://openalex.org/I134161618"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5132828998","display_name":"Tien-Hong Lo","orcid":null},"institutions":[{"id":"https://openalex.org/I134161618","display_name":"National Taiwan Normal University","ror":"https://ror.org/059dkdx38","country_code":"TW","type":"education","lineage":["https://openalex.org/I134161618"]}],"countries":["TW"],"is_corresponding":false,"raw_author_name":"Tien-Hong Lo","raw_affiliation_strings":["National Taiwan Normal University"],"affiliations":[{"raw_affiliation_string":"National Taiwan Normal University","institution_ids":["https://openalex.org/I134161618"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5132798217","display_name":"Yao-Ting Sung","orcid":null},"institutions":[{"id":"https://openalex.org/I134161618","display_name":"National Taiwan Normal University","ror":"https://ror.org/059dkdx38","country_code":"TW","type":"education","lineage":["https://openalex.org/I134161618"]}],"countries":["TW"],"is_corresponding":false,"raw_author_name":"Yao-Ting Sung","raw_affiliation_strings":["National Taiwan Normal University"],"affiliations":[{"raw_affiliation_string":"National Taiwan Normal University","institution_ids":["https://openalex.org/I134161618"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5132800680","display_name":"Berlin Chen","orcid":null},"institutions":[{"id":"https://openalex.org/I134161618","display_name":"National Taiwan Normal University","ror":"https://ror.org/059dkdx38","country_code":"TW","type":"education","lineage":["https://openalex.org/I134161618"]}],"countries":["TW"],"is_corresponding":false,"raw_author_name":"Berlin Chen","raw_affiliation_strings":["National Taiwan Normal University"],"affiliations":[{"raw_affiliation_string":"National Taiwan Normal University","institution_ids":["https://openalex.org/I134161618"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5132802095"],"corresponding_institution_ids":["https://openalex.org/I134161618"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.87580171,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"7"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.47200000286102295,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.47200000286102295,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10403","display_name":"Phonetics and Phonology Research","score":0.14489999413490295,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10667","display_name":"Emotion and Mood Recognition","score":0.09139999747276306,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/modality","display_name":"Modality (human\u2013computer interaction)","score":0.7574999928474426},{"id":"https://openalex.org/keywords/curriculum","display_name":"Curriculum","score":0.579800009727478},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.5383999943733215},{"id":"https://openalex.org/keywords/language-understanding","display_name":"Language understanding","score":0.47290000319480896},{"id":"https://openalex.org/keywords/component","display_name":"Component (thermodynamics)","score":0.38929998874664307},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.3386000096797943}],"concepts":[{"id":"https://openalex.org/C2780226545","wikidata":"https://www.wikidata.org/wiki/Q6888030","display_name":"Modality (human\u2013computer interaction)","level":2,"score":0.7574999928474426},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7258999943733215},{"id":"https://openalex.org/C47177190","wikidata":"https://www.wikidata.org/wiki/Q207137","display_name":"Curriculum","level":2,"score":0.579800009727478},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.5383999943733215},{"id":"https://openalex.org/C2983448237","wikidata":"https://www.wikidata.org/wiki/Q1078276","display_name":"Language understanding","level":2,"score":0.47290000319480896},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4700999855995178},{"id":"https://openalex.org/C168167062","wikidata":"https://www.wikidata.org/wiki/Q1117970","display_name":"Component (thermodynamics)","level":2,"score":0.38929998874664307},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.3386000096797943},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.29660001397132874},{"id":"https://openalex.org/C184337299","wikidata":"https://www.wikidata.org/wiki/Q1437428","display_name":"Semantics (computer science)","level":2,"score":0.2944999933242798},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.2904999852180481},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.289000004529953},{"id":"https://openalex.org/C2779903281","wikidata":"https://www.wikidata.org/wiki/Q6888026","display_name":"Modalities","level":2,"score":0.2727000117301941},{"id":"https://openalex.org/C2780660688","wikidata":"https://www.wikidata.org/wiki/Q25052564","display_name":"Multimodal learning","level":2,"score":0.27070000767707825},{"id":"https://openalex.org/C22367795","wikidata":"https://www.wikidata.org/wiki/Q7625208","display_name":"Structured prediction","level":2,"score":0.26669999957084656},{"id":"https://openalex.org/C2779439875","wikidata":"https://www.wikidata.org/wiki/Q1078276","display_name":"Natural language understanding","level":3,"score":0.2581999897956848}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/asru65441.2025.11434693","is_oa":false,"landing_page_url":"https://doi.org/10.1109/asru65441.2025.11434693","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 IEEE Automatic Speech Recognition and Understanding Workshop (ASRU)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","score":0.8697932362556458,"display_name":"Quality Education"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":11,"referenced_works":["https://openalex.org/W1991335324","https://openalex.org/W2148154194","https://openalex.org/W2251472752","https://openalex.org/W2296073425","https://openalex.org/W4223651314","https://openalex.org/W4308426870","https://openalex.org/W4319862401","https://openalex.org/W4386076642","https://openalex.org/W4391021508","https://openalex.org/W4406461704","https://openalex.org/W4407841907"],"related_works":[],"abstract_inverted_index":{"Traditional":[0],"Automated":[1],"Speaking":[2],"Assessment":[3],"(ASA)":[4],"systems":[5,119],"exhibit":[6],"inherent":[7],"modality":[8],"limitations:":[9],"text-based":[10],"approaches":[11],"lack":[12],"acoustic":[13],"information":[14],"while":[15],"audio-based":[16],"methods":[17],"miss":[18],"semantic":[19],"context.":[20],"Multimodal":[21,89],"Large":[22],"Language":[23],"Models":[24],"(MLLM)":[25],"offer":[26],"unprecedented":[27],"opportunities":[28],"for":[29,51,162],"comprehensive":[30,52],"ASA":[31],"by":[32],"simultaneously":[33],"processing":[34],"audio":[35],"and":[36,65],"text":[37],"within":[38],"unified":[39],"frameworks.":[40],"This":[41],"paper":[42],"presents":[43],"a":[44,93,114,127,159],"very":[45],"first":[46],"systematic":[47],"study":[48],"of":[49,58,63,103,111,130,141,150],"MLLM":[50,59],"ASA,":[53],"demonstrating":[54],"the":[55,61,71,122,139,142],"superior":[56],"performance":[57,125],"across":[60],"aspects":[62],"content":[64],"language":[66],"use.":[67],"However,":[68],"assessment":[69,124],"on":[70,113],"delivery":[72,143],"aspect":[73],"reveals":[74],"unique":[75],"challenges,":[76],"which":[77,156],"is":[78],"deemed":[79],"to":[80,97,132],"require":[81],"specialized":[82],"training":[83,154],"strategies.":[84],"We":[85],"thus":[86],"propose":[87],"Speech-First":[88],"Training":[90],"(SFMT),":[91],"leveraging":[92],"curriculum":[94],"learning":[95],"principle":[96],"establish":[98],"more":[99],"robust":[100],"modeling":[101],"foundations":[102],"speech":[104],"before":[105],"cross-modal":[106],"synergetic":[107],"fusion.":[108],"A":[109],"series":[110],"experiments":[112],"benchmark":[115],"dataset":[116],"show":[117],"MLLM-based":[118],"can":[120],"elevate":[121],"holistic":[123],"from":[126],"PCC":[128],"value":[129],"0.783":[131],"0.846.":[133],"In":[134],"particular,":[135],"SFMT":[136],"excels":[137],"in":[138],"evaluation":[140],"aspect,":[144],"achieving":[145],"an":[146],"absolute":[147],"accuracy":[148],"improvement":[149],"4%":[151],"over":[152],"conventional":[153],"approaches,":[155],"also":[157],"paves":[158],"new":[160],"avenue":[161],"ASA.":[163]},"counts_by_year":[],"updated_date":"2026-04-03T16:44:17.987007","created_date":"2026-04-03T00:00:00"}