{"id":"https://openalex.org/W4411900546","doi":"https://doi.org/10.59787/2413-5488-2025-50-2-18-31","title":"RELIABILITY OF AI IN FOREIGN LANGUAGE SPEAKING ASSESSMENT: COMPARING AUTOMATED AND HUMAN SCORING AMONG UNDERGRADUATE IT STUDENTS IN KAZAKHSTAN","display_name":"RELIABILITY OF AI IN FOREIGN LANGUAGE SPEAKING ASSESSMENT: COMPARING AUTOMATED AND HUMAN SCORING AMONG UNDERGRADUATE IT STUDENTS IN KAZAKHSTAN","publication_year":2025,"publication_date":"2025-06-01","ids":{"openalex":"https://openalex.org/W4411900546","doi":"https://doi.org/10.59787/2413-5488-2025-50-2-18-31"},"language":"en","primary_location":{"id":"doi:10.59787/2413-5488-2025-50-2-18-31","doi":"https://doi.org/10.59787/2413-5488-2025-50-2-18-31","is_oa":true,"landing_page_url":"https://doi.org/10.59787/2413-5488-2025-50-2-18-31","pdf_url":"https://highedujournal.kz/index.php/1/article/download/432/113/1832","source":{"id":"https://openalex.org/S4387290442","display_name":"National Center for Higher Education Development","issn_l":"2413-5488","issn":["2413-5488"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"National Center for Higher Education Development","raw_type":"journal-article"},"parent_works":[],"lineage":["https://openalex.org/W4411900546"],"child_works_count":0,"type":"article","is_traditional":true,"indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"diamond","oa_url":"https://highedujournal.kz/index.php/1/article/download/432/113/1832","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5023417916","display_name":"Zhibek Tleshova","orcid":"https://orcid.org/0000-0001-5095-5436","observed_orcids":["https://orcid.org/0000-0001-5095-5436"]},"institutions":[{"id":"https://openalex.org/I4405257690","display_name":"Astana IT University","ror":"https://ror.org/05jnvbc31","country_code":"KZ","type":"education","lineage":["https://openalex.org/I4405257690"]}],"countries":["KZ"],"is_corresponding":false,"raw_author_name":"Zhibek Tleshova","raw_affiliation_strings":["Astana IT University, Astana, Kazakhstan"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Astana IT University, Astana, Kazakhstan","institution_ids":["https://openalex.org/I4405257690"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5034977703","display_name":"Zhanar Tusselbayeva","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4405257690","display_name":"Astana IT University","ror":"https://ror.org/05jnvbc31","country_code":"KZ","type":"education","lineage":["https://openalex.org/I4405257690"]}],"countries":["KZ"],"is_corresponding":false,"raw_author_name":"Zhanar Tusselbayeva","raw_affiliation_strings":["Astana IT University, Astana, Kazakhstan"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Astana IT University, Astana, Kazakhstan","institution_ids":["https://openalex.org/I4405257690"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5118766881","display_name":"Aelita Ichshanova","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4405257690","display_name":"Astana IT University","ror":"https://ror.org/05jnvbc31","country_code":"KZ","type":"education","lineage":["https://openalex.org/I4405257690"]}],"countries":["KZ"],"is_corresponding":false,"raw_author_name":"Aelita Ichshanova","raw_affiliation_strings":["Astana IT University, Astana, Kazakhstan"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Astana IT University, Astana, Kazakhstan","institution_ids":["https://openalex.org/I4405257690"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5044280815","display_name":"Aigerim Urazbekova","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4405257690","display_name":"Astana IT University","ror":"https://ror.org/05jnvbc31","country_code":"KZ","type":"education","lineage":["https://openalex.org/I4405257690"]}],"countries":["KZ"],"is_corresponding":false,"raw_author_name":"Aigerim Urazbekova","raw_affiliation_strings":["Astana IT University, Astana, Kazakhstan"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Astana IT University, Astana, Kazakhstan","institution_ids":["https://openalex.org/I4405257690"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5118766882","display_name":"Meruyert Zhenisbayeva","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4405257690","display_name":"Astana IT University","ror":"https://ror.org/05jnvbc31","country_code":"KZ","type":"education","lineage":["https://openalex.org/I4405257690"]}],"countries":["KZ"],"is_corresponding":false,"raw_author_name":"Meruyert Zhenisbayeva","raw_affiliation_strings":["Astana IT University, Astana, Kazakhstan"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Astana IT University, Astana, Kazakhstan","institution_ids":["https://openalex.org/I4405257690"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5118766883","display_name":"Ali Orymbayev","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4403928190","display_name":"Astana International University","ror":"https://ror.org/0053vmf39","country_code":"KZ","type":"education","lineage":["https://openalex.org/I4403928190"]}],"countries":["KZ"],"is_corresponding":false,"raw_author_name":"Ali Orymbayev","raw_affiliation_strings":["Astana International University (AIU), Astana, Kazakhstan"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Astana International University (AIU), Astana, Kazakhstan","institution_ids":["https://openalex.org/I4403928190"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":1.577,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":{"value":0.81743333,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":91,"max":95},"biblio":{"volume":"2","issue":"50","first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10021","display_name":"EFL/ESL Teaching and Learning","score":0.7310000061988831,"subfield":{"id":"https://openalex.org/subfields/1203","display_name":"Language and Linguistics"},"field":{"id":"https://openalex.org/fields/12","display_name":"Arts and Humanities"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10021","display_name":"EFL/ESL Teaching and Learning","score":0.7310000061988831,"subfield":{"id":"https://openalex.org/subfields/1203","display_name":"Language and Linguistics"},"field":{"id":"https://openalex.org/fields/12","display_name":"Arts and Humanities"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T14516","display_name":"English Language Learning and Teaching","score":0.08089999854564667,"subfield":{"id":"https://openalex.org/subfields/3304","display_name":"Education"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T14414","display_name":"Artificial Intelligence in Education","score":0.03880000114440918,"subfield":{"id":"https://openalex.org/subfields/3304","display_name":"Education"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/automated-grading","display_name":"automated grading","score":0.7979999780654907},{"id":"https://openalex.org/keywords/human-scoring","display_name":"human scoring","score":0.7699999809265137},{"id":"https://openalex.org/keywords/inter-rater-reliability","display_name":"inter-rater reliability","score":0.925000011920929},{"id":"https://openalex.org/keywords/intra-rater-reliability","display_name":"intra-rater reliability","score":0.9959999918937683},{"id":"https://openalex.org/keywords/english-as-a-foreign-language","display_name":"English as a foreign language","score":0.9139999747276306},{"id":"https://openalex.org/keywords/hybrid-assessment","display_name":"hybrid evaluation","score":0.6349999904632568},{"id":"https://openalex.org/keywords/rubrics","display_name":"rubrics","score":0.6349999904632568},{"id":"https://openalex.org/keywords/rater-training","display_name":"rater training","score":0.9369999766349792},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"speech recognition","score":0.7210000157356262},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"natural language processing","score":0.8820000290870667},{"id":"https://openalex.org/keywords/algorithmic-bias","display_name":"algorithmic bias","score":0.41499999165534973}],"concepts":[{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.7176233530044556},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.49543964862823486},{"id":"https://openalex.org/C166007726","wikidata":"https://www.wikidata.org/wiki/Q4536542","display_name":"Language assessment","level":2,"score":0.48964911699295044},{"id":"https://openalex.org/C114010052","wikidata":"https://www.wikidata.org/wiki/Q150352","display_name":"Foreign language","level":2,"score":0.45195311307907104},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.4482663571834564},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.4399097263813019},{"id":"https://openalex.org/C145420912","wikidata":"https://www.wikidata.org/wiki/Q853077","display_name":"Mathematics education","level":1,"score":0.41955554485321045},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3411727547645569},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C163258240","wikidata":"https://www.wikidata.org/wiki/Q25342","display_name":"Power (physics)","level":2,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.59787/2413-5488-2025-50-2-18-31","doi":"https://doi.org/10.59787/2413-5488-2025-50-2-18-31","is_oa":true,"landing_page_url":"https://doi.org/10.59787/2413-5488-2025-50-2-18-31","pdf_url":"https://highedujournal.kz/index.php/1/article/download/432/113/1832","source":{"id":"https://openalex.org/S4387290442","display_name":"National Center for Higher Education Development","issn_l":"2413-5488","issn":["2413-5488"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"National Center for Higher Education Development","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.59787/2413-5488-2025-50-2-18-31","doi":"https://doi.org/10.59787/2413-5488-2025-50-2-18-31","is_oa":true,"landing_page_url":"https://doi.org/10.59787/2413-5488-2025-50-2-18-31","pdf_url":"https://highedujournal.kz/index.php/1/article/download/432/113/1832","source":{"id":"https://openalex.org/S4387290442","display_name":"National Center for Higher Education Development","issn_l":"2413-5488","issn":["2413-5488"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"National Center for Higher Education Development","raw_type":"journal-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","score":0.6164000034332275,"display_name":"Quality education"}],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4411900546.pdf","grobid_xml":"https://content.openalex.org/works/W4411900546.grobid-xml"},"referenced_works_count":21,"referenced_works":["https://openalex.org/W751068977","https://openalex.org/W1531673065","https://openalex.org/W1587080882","https://openalex.org/W2113523640","https://openalex.org/W2322695415","https://openalex.org/W2477907264","https://openalex.org/W2594882268","https://openalex.org/W2907416181","https://openalex.org/W3012074514","https://openalex.org/W3013082551","https://openalex.org/W3175012739","https://openalex.org/W3202338402","https://openalex.org/W4282569893","https://openalex.org/W4294214983","https://openalex.org/W4307047732","https://openalex.org/W4312117896","https://openalex.org/W4376625966","https://openalex.org/W4386613783","https://openalex.org/W4396850518","https://openalex.org/W4407062036","https://openalex.org/W4413183906"],"related_works":["https://openalex.org/W7207588040","https://openalex.org/W4412717669","https://openalex.org/W7214314339","https://openalex.org/W7147387452","https://openalex.org/W4413076222","https://openalex.org/W7221473790","https://openalex.org/W4407035080","https://openalex.org/W7202095984","https://openalex.org/W7171623514","https://openalex.org/W4408951576"],"abstract_inverted_index":{"The":[0,97],"integration":[1],"of":[2,30,60,79],"Artificial":[3],"Intelligence":[4],"(AI)":[5],"in":[6,10,23,46,62,122,134],"language":[7,139],"assessment,":[8],"particularly":[9],"evaluating":[11],"speaking":[12,32,146],"skills,":[13],"has":[14],"introduced":[15],"opportunities":[16],"for":[17,69,102],"greater":[18],"consistency,":[19],"efficiency,":[20],"and":[21,43,58,77,87,94,107,131,137,152],"scalability":[22],"educational":[24,153],"contexts.":[25],"This":[26,81,114,141],"paper":[27],"studiesthe":[28],"reliability":[29,45],"AI-assisted":[31],"assessment":[33,120],"compared":[34],"to":[35,74,91,111,144],"human-mediated":[36],"evaluation,":[37],"with":[38],"a":[39,49,118],"focus":[40],"on":[41],"inter-rater":[42],"intra-rater":[44],"English":[47],"as":[48,66],"Foreign":[50],"Language":[51],"(EFL)":[52],"learning.":[53],"Thispaper":[54],"explores":[55],"the":[56,100],"strengths":[57],"limitations":[59],"AI":[61,88,108,124],"automated":[63],"scoring,":[64],"such":[65],"its":[67],"capacity":[68],"standardization,":[70],"alongside":[71],"challenges":[72],"related":[73],"validity,":[75],"bias,":[76],"interpretability":[78],"results.":[80],"study":[82,98],"reviews":[83],"discrepancies":[84],"between":[85],"human":[86,126],"scoring":[89],"due":[90],"subjective":[92],"judgment":[93],"training":[95],"limitations.":[96],"emphasizes":[99],"need":[101],"standardized":[103],"rubrics,":[104],"rater":[105],"training,":[106],"model":[109],"calibration":[110],"enhance":[112],"reliability.":[113],"paperconcludes":[115],"by":[116,129],"proposing":[117],"hybrid":[119],"framework":[121],"which":[123],"complements":[125],"raters,":[127],"supported":[128],"methodological":[130],"technical":[132],"improvements":[133],"speech":[135],"recognition":[136],"natural":[138],"processing.":[140],"approach":[142],"aims":[143],"optimize":[145],"proficiency":[147],"evaluations":[148],"while":[149],"maintaining":[150],"fairness":[151],"integrity.":[154]},"counts_by_year":[{"year":2026,"cited_by_count":1}],"updated_date":"2026-11-13T13:42:25.865203","created_date":"2025-10-10T00:00:00"}