{"id":"https://openalex.org/W2011522158","doi":"https://doi.org/10.1002/j.2333-8504.2009.tb02188.x","title":"HOW DO RATERS FROM INDIA PERFORM IN SCORING THE TOEFL IBT\u2122 SPEAKING SECTION AND WHAT KIND OF TRAINING HELPS?","display_name":"HOW DO RATERS FROM INDIA PERFORM IN SCORING THE TOEFL IBT\u2122 SPEAKING SECTION AND WHAT KIND OF TRAINING HELPS?","publication_year":2009,"publication_date":"2009-12-01","ids":{"openalex":"https://openalex.org/W2011522158","doi":"https://doi.org/10.1002/j.2333-8504.2009.tb02188.x","mag":"2011522158"},"language":"en","primary_location":{"id":"doi:10.1002/j.2333-8504.2009.tb02188.x","doi":"https://doi.org/10.1002/j.2333-8504.2009.tb02188.x","is_oa":true,"landing_page_url":"https://doi.org/10.1002/j.2333-8504.2009.tb02188.x","pdf_url":"https://onlinelibrary.wiley.com/doi/pdfdirect/10.1002/j.2333-8504.2009.tb02188.x","source":{"id":"https://openalex.org/S2764556220","display_name":"ETS Research Report Series","issn_l":"2330-8516","issn":["2330-8516"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","poland-20","russia-white-list-3"],"host_organization":"https://openalex.org/P4310320595","host_organization_name":"Wiley","host_organization_lineage":["https://openalex.org/P4310320595"],"host_organization_lineage_names":["Wiley"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ETS Research Report Series","raw_type":"journal-article"},"volume":null,"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"bronze","oa_url":"https://onlinelibrary.wiley.com/doi/pdfdirect/10.1002/j.2333-8504.2009.tb02188.x","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Xiaoming Xi","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I1341030882","display_name":"Educational Testing Service","ror":"https://ror.org/03b5q4637","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I1341030882"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Xiaoming Xi","raw_affiliation_strings":["ETS, Princeton, New Jersey"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ETS, Princeton, New Jersey","institution_ids":["https://openalex.org/I1341030882"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5058211012","display_name":"Pam Mollaun","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I1341030882","display_name":"Educational Testing Service","ror":"https://ror.org/03b5q4637","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I1341030882"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Pam Mollaun","raw_affiliation_strings":["ETS, Princeton, New Jersey"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ETS, Princeton, New Jersey","institution_ids":["https://openalex.org/I1341030882"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I1341030882"],"apc_list":null,"apc_paid":null,"fwci":19.7926,"has_fulltext":true,"cited_by_count":26,"citation_normalized_percentile":{"value":0.9874249,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":88,"max":98},"biblio":{"volume":"2009","issue":"2","first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10021","display_name":"EFL/ESL Teaching and Learning","score":0.8574000000953674,"subfield":{"id":"https://openalex.org/subfields/1203","display_name":"Language and Linguistics"},"field":{"id":"https://openalex.org/fields/12","display_name":"Arts and Humanities"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10021","display_name":"EFL/ESL Teaching and Learning","score":0.8574000000953674,"subfield":{"id":"https://openalex.org/subfields/1203","display_name":"Language and Linguistics"},"field":{"id":"https://openalex.org/fields/12","display_name":"Arts and Humanities"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10467","display_name":"Psychometric Methodologies and Testing","score":0.09690000116825104,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10959","display_name":"Student Assessment and Feedback","score":0.004800000227987766,"subfield":{"id":"https://openalex.org/subfields/3304","display_name":"Education"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/toefl-ibt","display_name":"TOEFL iBT","score":0.9020000100135803},{"id":"https://openalex.org/keywords/rater-training","display_name":"rater training","score":0.8980000019073486},{"id":"https://openalex.org/keywords/rater-reliability","display_name":"rater reliability","score":0.6629999876022339},{"id":"https://openalex.org/keywords/score-reliability","display_name":"scoring reliability","score":0.5379999876022339}],"concepts":[{"id":"https://openalex.org/C18563106","wikidata":"https://www.wikidata.org/wiki/Q487425","display_name":"Test of English as a Foreign Language","level":3,"score":0.8602116703987122},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.6119564175605774},{"id":"https://openalex.org/C46304622","wikidata":"https://www.wikidata.org/wiki/Q374814","display_name":"Certification","level":2,"score":0.6088652610778809},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.6088361740112305},{"id":"https://openalex.org/C2779182362","wikidata":"https://www.wikidata.org/wiki/Q17126187","display_name":"Session (web analytics)","level":2,"score":0.5594106316566467},{"id":"https://openalex.org/C509550671","wikidata":"https://www.wikidata.org/wiki/Q126945","display_name":"Medical education","level":1,"score":0.39794573187828064},{"id":"https://openalex.org/C166007726","wikidata":"https://www.wikidata.org/wiki/Q4536542","display_name":"Language assessment","level":2,"score":0.2943696677684784},{"id":"https://openalex.org/C145420912","wikidata":"https://www.wikidata.org/wiki/Q853077","display_name":"Mathematics education","level":1,"score":0.2876136302947998},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.18523314595222473},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.18370944261550903},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1002/j.2333-8504.2009.tb02188.x","doi":"https://doi.org/10.1002/j.2333-8504.2009.tb02188.x","is_oa":true,"landing_page_url":"https://doi.org/10.1002/j.2333-8504.2009.tb02188.x","pdf_url":"https://onlinelibrary.wiley.com/doi/pdfdirect/10.1002/j.2333-8504.2009.tb02188.x","source":{"id":"https://openalex.org/S2764556220","display_name":"ETS Research Report Series","issn_l":"2330-8516","issn":["2330-8516"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","poland-20","russia-white-list-3"],"host_organization":"https://openalex.org/P4310320595","host_organization_name":"Wiley","host_organization_lineage":["https://openalex.org/P4310320595"],"host_organization_lineage_names":["Wiley"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ETS Research Report Series","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1002/j.2333-8504.2009.tb02188.x","doi":"https://doi.org/10.1002/j.2333-8504.2009.tb02188.x","is_oa":true,"landing_page_url":"https://doi.org/10.1002/j.2333-8504.2009.tb02188.x","pdf_url":"https://onlinelibrary.wiley.com/doi/pdfdirect/10.1002/j.2333-8504.2009.tb02188.x","source":{"id":"https://openalex.org/S2764556220","display_name":"ETS Research Report Series","issn_l":"2330-8516","issn":["2330-8516"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","poland-20","russia-white-list-3"],"host_organization":"https://openalex.org/P4310320595","host_organization_name":"Wiley","host_organization_lineage":["https://openalex.org/P4310320595"],"host_organization_lineage_names":["Wiley"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ETS Research Report Series","raw_type":"journal-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","display_name":"Quality education","score":0.6164000034332275}],"sustainable_development_goals_aurora":[{"id":"https://metadata.un.org/sdg/4","display_name":"Quality education","score":0.7799999713897705}],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2011522158.pdf","grobid_xml":"https://content.openalex.org/works/W2011522158.grobid-xml"},"referenced_works_count":34,"referenced_works":["https://openalex.org/W357129868","https://openalex.org/W1967407004","https://openalex.org/W1968467642","https://openalex.org/W1972794608","https://openalex.org/W1980081737","https://openalex.org/W1982628073","https://openalex.org/W2006170361","https://openalex.org/W2006559300","https://openalex.org/W2010821160","https://openalex.org/W2017665304","https://openalex.org/W2021591245","https://openalex.org/W2022389086","https://openalex.org/W2037789405","https://openalex.org/W2043544836","https://openalex.org/W2045813592","https://openalex.org/W2050460112","https://openalex.org/W2053154970","https://openalex.org/W2054579521","https://openalex.org/W2068511252","https://openalex.org/W2070471368","https://openalex.org/W2082767250","https://openalex.org/W2086996448","https://openalex.org/W2087274684","https://openalex.org/W2111266316","https://openalex.org/W2113523640","https://openalex.org/W2127051216","https://openalex.org/W2139403472","https://openalex.org/W2143149791","https://openalex.org/W2143536002","https://openalex.org/W2168303754","https://openalex.org/W2328301182","https://openalex.org/W2553721728","https://openalex.org/W2588592377","https://openalex.org/W4238804190"],"related_works":["https://openalex.org/W1609477948","https://openalex.org/W2155314571","https://openalex.org/W181659254","https://openalex.org/W2000641383","https://openalex.org/W2245552235","https://openalex.org/W1997436821","https://openalex.org/W2951549985","https://openalex.org/W2148011386","https://openalex.org/W2294237342","https://openalex.org/W2402347362"],"abstract_inverted_index":{"ABSTRACT":[0],"This":[1],"study":[2],"investigated":[3],"the":[4,7,36,52,116,126,144,153,157,174,190,198,210,214],"scoring":[5,83,111,125,146,202,235],"of":[6,9,26,76,92,232],"Test":[8,16],"English":[10,27],"as":[11,88,195,197],"a":[12,77,163],"Foreign":[13],"Language\u2122":[14],"Internet\u2010based":[15],"(TOEFL":[17],"iBT\u2122)":[18],"Speaking":[19,53,129],"section":[20,54],"by":[21,138],"bilingual":[22],"or":[23,30],"multilingual":[24],"speakers":[25],"and":[28,46,72,95,106,205],"1":[29,148],"more":[31,219],"Indian":[32,69,84,204,217,236],"languages.":[33],"We":[34],"explored":[35],"extent":[37],"to":[38,50,135,151,173,183,222],"which":[39,131,185],"raters":[40,97,139,188,191,200,215],"from":[41,98,192],"India,":[42],"after":[43],"being":[44],"trained":[45,94,161],"certified,":[47],"were":[48,100,170],"able":[49],"score":[51,216,224],"for":[55,82,124],"TOEFL":[56,127],"iBT":[57,128],"examinees":[58,85,218],"with":[59,180],"mixed":[60],"first":[61,117],"language":[62],"(L1)":[63],"backgrounds,":[64],"especially":[65],"those":[66],"speaking":[67],"an":[68],"language,":[70],"accurately":[71],"consistently.":[73],"The":[74],"effectiveness":[75],"special":[78,164,211],"training":[79,123,155,165,181,212],"package":[80],"designed":[81],"was":[86,132,160,177],"examined":[87],"well.":[89],"A":[90],"total":[91],"26":[93],"certified":[96],"India":[99,193],"randomly":[101],"divided":[102],"into":[103],"2":[104,109],"groups":[105,120],"participated":[107],"in":[108,113,140,201,234],"on\u2010site":[110],"sessions":[112],"Mumbai.":[114],"In":[115,143,208],"session,":[118,147],"both":[119,203],"received":[121,137],"regular":[122,154],"section,":[130],"largely":[133],"similar":[134,182],"that":[136,179,184],"North":[141],"America.":[142],"second":[145,158],"group":[149,159],"continued":[150],"receive":[152],"while":[156],"using":[162],"package.":[166],"Rater":[167],"feedback":[168],"surveys":[169],"also":[171,228],"given":[172],"raters.":[175],"It":[176,227],"found":[178],"operational":[186,199],"U.S.\u2010based":[187],"receive,":[189],"performed":[194],"well":[196],"non\u2010Indian":[206],"examinees.":[207,237],"addition,":[209],"helped":[213],"consistently,":[220],"leading":[221],"increased":[223],"reliability":[225],"estimates.":[226],"boosted":[229],"raters'":[230],"levels":[231],"confidence":[233]},"counts_by_year":[{"year":2024,"cited_by_count":1},{"year":2023,"cited_by_count":2},{"year":2022,"cited_by_count":2},{"year":2021,"cited_by_count":1},{"year":2020,"cited_by_count":2},{"year":2019,"cited_by_count":2},{"year":2018,"cited_by_count":1},{"year":2017,"cited_by_count":2},{"year":2016,"cited_by_count":1},{"year":2015,"cited_by_count":3},{"year":2014,"cited_by_count":2},{"year":2012,"cited_by_count":5}],"updated_date":"2026-10-30T12:39:25.321706","created_date":"2016-06-24T00:00:00"}