{"id":"https://openalex.org/W7225655079","doi":"https://doi.org/10.3390/jintelligence14110296","title":"Human and AI Judgment in EFL Speaking Assessment: Psychometric Dependability, Feedback Quantity and Coverage, and Exploratory Teacher Perspectives","display_name":"Human and AI Judgment in EFL Speaking Assessment: Psychometric Dependability, Feedback Quantity and Coverage, and Exploratory Teacher Perspectives","publication_year":2026,"publication_date":"2026-11-09","ids":{"openalex":"https://openalex.org/W7225655079","doi":"https://doi.org/10.3390/jintelligence14110296"},"language":"en","primary_location":{"id":"doi:10.3390/jintelligence14110296","doi":"https://doi.org/10.3390/jintelligence14110296","is_oa":true,"landing_page_url":"https://doi.org/10.3390/jintelligence14110296","pdf_url":null,"source":{"id":"https://openalex.org/S2737368440","display_name":"Journal of Intelligence","issn_l":"2079-3200","issn":["2079-3200"],"is_oa":true,"is_in_doaj":true,"is_core":true,"listed_in":["anvur-scientific","cwts-core","doaj","erih-plus","norway-1","poland-40","russia-white-list-2","vabb-shw"],"host_organization":"https://openalex.org/P4310310987","host_organization_name":"Multidisciplinary Digital Publishing Institute","host_organization_lineage":["https://openalex.org/P4310310987"],"host_organization_lineage_names":["Multidisciplinary Digital Publishing Institute"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of Intelligence","raw_type":"journal-article"},"volume":null,"type":"article","is_traditional":true,"indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.3390/jintelligence14110296","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5148802345","display_name":"Yan Zhang","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4405271820","display_name":"Jiaxing Nanhu University","ror":"https://ror.org/033zv7b52","country_code":"CN","type":"education","lineage":["https://openalex.org/I4405271820"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yan Zhang","raw_affiliation_strings":["Department of Public Basic Education, Jiaxing Nanhu University, Jiaxing 314001, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Public Basic Education, Jiaxing Nanhu University, Jiaxing 314001, China","institution_ids":["https://openalex.org/I4405271820"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5044277116","display_name":"Jinyan Huang","orcid":"https://orcid.org/0000-0003-1641-6580","observed_orcids":["https://orcid.org/0000-0003-1641-6580"]},"institutions":[{"id":"https://openalex.org/I135237710","display_name":"Zhejiang Normal University","ror":"https://ror.org/01vevwk45","country_code":"CN","type":"education","lineage":["https://openalex.org/I135237710"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Jinyan Huang","raw_affiliation_strings":["College of Education, Zhejiang Normal University, Jinhua 321004, China"],"raw_orcid":"https://orcid.org/0000-0003-1641-6580","affiliations":[{"raw_affiliation_string":"College of Education, Zhejiang Normal University, Jinhua 321004, China","institution_ids":["https://openalex.org/I135237710"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5161338109","display_name":"Jun Yan","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I142823887","display_name":"St. John's University","ror":"https://ror.org/00bgtad15","country_code":"US","type":"education","lineage":["https://openalex.org/I142823887"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jun Yan","raw_affiliation_strings":["Department of Education Specialties, St. John\u2019s University, New York, NY 11439, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Education Specialties, St. John\u2019s University, New York, NY 11439, USA","institution_ids":["https://openalex.org/I142823887"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5044277116"],"corresponding_institution_ids":["https://openalex.org/I135237710"],"apc_list":{"value":2600,"currency":"CHF","value_usd":3137},"apc_paid":{"value":2600,"currency":"CHF","value_usd":3137},"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.0,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"14","issue":"11","first_page":"296","last_page":"296"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10021","display_name":"EFL/ESL Teaching and Learning","score":0.8438000082969666,"subfield":{"id":"https://openalex.org/subfields/1203","display_name":"Language and Linguistics"},"field":{"id":"https://openalex.org/fields/12","display_name":"Arts and Humanities"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10021","display_name":"EFL/ESL Teaching and Learning","score":0.8438000082969666,"subfield":{"id":"https://openalex.org/subfields/1203","display_name":"Language and Linguistics"},"field":{"id":"https://openalex.org/fields/12","display_name":"Arts and Humanities"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10959","display_name":"Student Assessment and Feedback","score":0.07980000227689743,"subfield":{"id":"https://openalex.org/subfields/3304","display_name":"Education"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T14516","display_name":"English Language Learning and Teaching","score":0.01850000023841858,"subfield":{"id":"https://openalex.org/subfields/3304","display_name":"Education"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/gpt-4o","display_name":"GPT-4o","score":0.9459999799728394},{"id":"https://openalex.org/keywords/rater-reliability","display_name":"rater reliability","score":0.5389999747276306},{"id":"https://openalex.org/keywords/generalizability-theory","display_name":"generalizability theory","score":0.7960000038146973},{"id":"https://openalex.org/keywords/teacher-perceptions","display_name":"teacher perceptions","score":0.7089999914169312},{"id":"https://openalex.org/keywords/individual-differences","display_name":"individual differences","score":0.6209999918937683},{"id":"https://openalex.org/keywords/chinese-college-students","display_name":"Chinese university students","score":0.5199999809265137}],"concepts":[{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.8439000248908997},{"id":"https://openalex.org/C2777413886","wikidata":"https://www.wikidata.org/wiki/Q3276013","display_name":"Fluency","level":2,"score":0.6384000182151794},{"id":"https://openalex.org/C164850336","wikidata":"https://www.wikidata.org/wiki/Q3685487","display_name":"Interpersonal communication","level":2,"score":0.5576000213623047},{"id":"https://openalex.org/C2779182362","wikidata":"https://www.wikidata.org/wiki/Q17126187","display_name":"Session (web analytics)","level":2,"score":0.54339998960495},{"id":"https://openalex.org/C85973986","wikidata":"https://www.wikidata.org/wiki/Q1091731","display_name":"Exploratory research","level":2,"score":0.44290000200271606},{"id":"https://openalex.org/C174107131","wikidata":"https://www.wikidata.org/wiki/Q191591","display_name":"Emotional intelligence","level":2,"score":0.3995000123977661},{"id":"https://openalex.org/C74196892","wikidata":"https://www.wikidata.org/wiki/Q7781188","display_name":"Thematic analysis","level":3,"score":0.38530001044273376},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.37709999084472656},{"id":"https://openalex.org/C145420912","wikidata":"https://www.wikidata.org/wiki/Q853077","display_name":"Mathematics education","level":1,"score":0.37139999866485596},{"id":"https://openalex.org/C75630572","wikidata":"https://www.wikidata.org/wiki/Q538904","display_name":"Applied psychology","level":1,"score":0.3709999918937683},{"id":"https://openalex.org/C171606756","wikidata":"https://www.wikidata.org/wiki/Q506132","display_name":"Psychometrics","level":2,"score":0.3617999851703644},{"id":"https://openalex.org/C91034043","wikidata":"https://www.wikidata.org/wiki/Q223642","display_name":"Interpersonal relationship","level":2,"score":0.3264999985694885},{"id":"https://openalex.org/C118147538","wikidata":"https://www.wikidata.org/wiki/Q1126970","display_name":"Metacognition","level":3,"score":0.32600000500679016},{"id":"https://openalex.org/C104709138","wikidata":"https://www.wikidata.org/wiki/Q1671540","display_name":"Intraclass correlation","level":3,"score":0.3190999925136566},{"id":"https://openalex.org/C102959455","wikidata":"https://www.wikidata.org/wiki/Q7313954","display_name":"Repeated measures design","level":2,"score":0.29589998722076416},{"id":"https://openalex.org/C86772948","wikidata":"https://www.wikidata.org/wiki/Q1413287","display_name":"Social skills","level":2,"score":0.29429998993873596},{"id":"https://openalex.org/C138496976","wikidata":"https://www.wikidata.org/wiki/Q175002","display_name":"Developmental psychology","level":1,"score":0.2757999897003174},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.2754000127315521},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.2662999927997589}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.3390/jintelligence14110296","doi":"https://doi.org/10.3390/jintelligence14110296","is_oa":true,"landing_page_url":"https://doi.org/10.3390/jintelligence14110296","pdf_url":null,"source":{"id":"https://openalex.org/S2737368440","display_name":"Journal of Intelligence","issn_l":"2079-3200","issn":["2079-3200"],"is_oa":true,"is_in_doaj":true,"is_core":true,"listed_in":["anvur-scientific","cwts-core","doaj","erih-plus","norway-1","poland-40","russia-white-list-2","vabb-shw"],"host_organization":"https://openalex.org/P4310310987","host_organization_name":"Multidisciplinary Digital Publishing Institute","host_organization_lineage":["https://openalex.org/P4310310987"],"host_organization_lineage_names":["Multidisciplinary Digital Publishing Institute"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of Intelligence","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.3390/jintelligence14110296","doi":"https://doi.org/10.3390/jintelligence14110296","is_oa":true,"landing_page_url":"https://doi.org/10.3390/jintelligence14110296","pdf_url":null,"source":{"id":"https://openalex.org/S2737368440","display_name":"Journal of Intelligence","issn_l":"2079-3200","issn":["2079-3200"],"is_oa":true,"is_in_doaj":true,"is_core":true,"listed_in":["anvur-scientific","cwts-core","doaj","erih-plus","norway-1","poland-40","russia-white-list-2","vabb-shw"],"host_organization":"https://openalex.org/P4310310987","host_organization_name":"Multidisciplinary Digital Publishing Institute","host_organization_lineage":["https://openalex.org/P4310310987"],"host_organization_lineage_names":["Multidisciplinary Digital Publishing Institute"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of Intelligence","raw_type":"journal-article"},"sustainable_development_goals":[{"display_name":"Quality education","score":0.8278999924659729,"id":"https://metadata.un.org/sdg/4"}],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":28,"referenced_works":["https://openalex.org/W1998446689","https://openalex.org/W2067724039","https://openalex.org/W2068591858","https://openalex.org/W2095176121","https://openalex.org/W2127746072","https://openalex.org/W2327037637","https://openalex.org/W2481953871","https://openalex.org/W2560140854","https://openalex.org/W2801630847","https://openalex.org/W3036190796","https://openalex.org/W4238804190","https://openalex.org/W4241187983","https://openalex.org/W4301140379","https://openalex.org/W4308426870","https://openalex.org/W4378193514","https://openalex.org/W4379193309","https://openalex.org/W4385293522","https://openalex.org/W4386253646","https://openalex.org/W4389793309","https://openalex.org/W4390821180","https://openalex.org/W4393376882","https://openalex.org/W4410811404","https://openalex.org/W4410895495","https://openalex.org/W4411379457","https://openalex.org/W4412003972","https://openalex.org/W4413999160","https://openalex.org/W4415483010","https://openalex.org/W7171623514"],"related_works":["https://openalex.org/W4413076222","https://openalex.org/W4413841833","https://openalex.org/W7214314339","https://openalex.org/W7117648786","https://openalex.org/W7138963029","https://openalex.org/W7126123778","https://openalex.org/W7207588040","https://openalex.org/W4400138640","https://openalex.org/W7162078440","https://openalex.org/W7117577393"],"abstract_inverted_index":{"This":[0],"study":[1,221],"examined":[2],"human-AI":[3,233],"judgment":[4],"in":[5,31,184,194,243],"EFL":[6,49,246],"speaking":[7,247],"assessment":[8],"as":[9,98,251],"a":[10,32,151],"measurement":[11,231],"problem":[12],"relevant":[13],"to":[14,123,136,142,223],"research":[15,225],"on":[16],"individual":[17,241],"differences:":[18],"how":[19],"consistently":[20],"do":[21],"expert":[22],"raters":[23],"and":[24,51,62,73,76,82,88,117,125,140,148,191,197,217,232,255],"repeated":[25,56,162],"AI":[26,187],"sessions":[27,57,164],"preserve":[28,239],"between-person":[29],"differences":[30,242],"complex":[33,244],"performance?":[34],"Thirty":[35],"oral-performance":[36],"samples":[37],"from":[38,134,175,262],"first-year":[39],"Chinese":[40],"university":[41],"English":[42],"majors":[43],"were":[44,67,85,96,111,259],"evaluated":[45,78],"by":[46,52],"four":[47,55,94,128,161,198,210],"experienced":[48],"teachers":[50],"ChatGPT-4o":[53,115,139,163,201],"across":[54],"using":[58],"the":[59,92,181,209,227,263],"same":[60,93],"holistic":[61,147,177,182],"accuracy-fluency-complexity":[63],"framework.":[64],"Score":[65],"distributions":[66],"summarized":[68,86],"descriptively;":[69],"human-rater":[70],"intraclass":[71],"correlations":[72],"generalizability-theory":[74],"G-":[75],"D-studies":[77],"dependability.":[79],"Feedback":[80],"quantity":[81],"thematic":[83],"coverage":[84],"descriptively,":[87,207],"post-results":[89],"interviews":[90],"with":[91,127,160],"teacher-raters":[95,211],"analyzed":[97],"exploratory":[99],"participant":[100],"reflections.":[101],"Teachers":[102],"produced":[103,203],"more":[104,204],"dependable":[105],"single":[106],"ratings.":[107],"Holistic":[108],"phi":[109,152],"coefficients":[110,132],"0.54":[112,135],"for":[113,119,138,144,215],"one":[114,120],"session":[116],"0.65":[118,143],"teacher,":[121],"increasing":[122],"0.83":[124],"0.88":[126],"assessments.":[129],"Analytic":[130],"single-assessment":[131],"ranged":[133],"0.56":[137],"0.61":[141],"teachers.":[145,168],"Across":[146],"analytic":[149,170],"scoring,":[150],"coefficient":[153],"of":[154,229],"at":[155,226],"least":[156],"0.80":[157],"was":[158,248],"reached":[159],"but":[165],"three":[166,196],"independent":[167],"ChatGPT-4o\u2019s":[169],"dimensions":[171],"showed":[172],"limited":[173],"differentiation":[174],"its":[176],"scoring:":[178],"accuracy":[179],"matched":[180],"score":[183,264],"all":[185],"120":[186],"ratings,":[188,199],"while":[189,208],"fluency":[190],"complexity":[192],"differed":[193],"only":[195],"respectively.":[200],"also":[202],"rubric-relevant":[205],"comments":[206],"emphasized":[212],"human":[213],"oversight":[214],"contextual":[216],"interpersonal":[218],"interpretation.":[219],"The":[220],"contributes":[222],"intelligence":[224,253],"level":[228],"psychometric":[230],"judgment:":[234],"assessor":[235],"systems":[236],"can":[237],"differentially":[238],"observed":[240],"performance.":[245],"not":[249,260],"treated":[250],"an":[252],"test,":[254],"unmeasured":[256],"cognitive":[257],"mechanisms":[258],"inferred":[261],"patterns.":[265]},"counts_by_year":[],"updated_date":"2026-11-11T12:16:26.678958","created_date":"2026-11-10T00:00:00"}