{"id":"https://openalex.org/W3211915243","doi":"https://doi.org/10.1186/s40468-021-00134-5","title":"The affectability of writing assessment scores: a G-theory analysis of rater, task, and scoring method contribution","display_name":"The affectability of writing assessment scores: a G-theory analysis of rater, task, and scoring method contribution","publication_year":2021,"publication_date":"2021-10-01","ids":{"openalex":"https://openalex.org/W3211915243","doi":"https://doi.org/10.1186/s40468-021-00134-5","mag":"3211915243"},"language":"en","primary_location":{"id":"doi:10.1186/s40468-021-00134-5","doi":"https://doi.org/10.1186/s40468-021-00134-5","is_oa":true,"landing_page_url":"https://doi.org/10.1186/s40468-021-00134-5","pdf_url":"https://languagetestingasia.springeropen.com/track/pdf/10.1186/s40468-021-00134-5","source":{"id":"https://openalex.org/S2736419388","display_name":"Language Testing in Asia","issn_l":"2229-0443","issn":["2229-0443"],"is_oa":true,"is_in_doaj":true,"is_core":true,"listed_in":["cwts-core","doaj","erih-plus","jufo-1","norway-1","poland-20","russia-white-list-1"],"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Language Testing in Asia","raw_type":"journal-article"},"volume":null,"type":"article","is_traditional":true,"indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://languagetestingasia.springeropen.com/track/pdf/10.1186/s40468-021-00134-5","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5041292769","display_name":"Ali Khodi","orcid":"https://orcid.org/0000-0002-5691-5387","observed_orcids":["https://orcid.org/0000-0002-5691-5387"]},"institutions":[{"id":"https://openalex.org/I1312987727","display_name":"University of Neyshabur","ror":"https://ror.org/01x41eb05","country_code":"IR","type":"education","lineage":["https://openalex.org/I1312987727"]}],"countries":["IR"],"is_corresponding":true,"raw_author_name":"Ali Khodi","raw_affiliation_strings":["University of Neyshabur, Neyshabur, Iran"],"raw_orcid":"https://orcid.org/0000-0002-5691-5387","affiliations":[{"raw_affiliation_string":"University of Neyshabur, Neyshabur, Iran","institution_ids":["https://openalex.org/I1312987727"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5041292769"],"corresponding_institution_ids":["https://openalex.org/I1312987727"],"apc_list":{"value":1390,"currency":"USD","value_usd":1390},"apc_paid":{"value":1390,"currency":"USD","value_usd":1390},"fwci":7.7725,"has_fulltext":true,"cited_by_count":19,"citation_normalized_percentile":{"value":0.9694177,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":88,"max":99},"biblio":{"volume":"11","issue":"1","first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10021","display_name":"EFL/ESL Teaching and Learning","score":0.7592999935150146,"subfield":{"id":"https://openalex.org/subfields/1203","display_name":"Language and Linguistics"},"field":{"id":"https://openalex.org/fields/12","display_name":"Arts and Humanities"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10021","display_name":"EFL/ESL Teaching and Learning","score":0.7592999935150146,"subfield":{"id":"https://openalex.org/subfields/1203","display_name":"Language and Linguistics"},"field":{"id":"https://openalex.org/fields/12","display_name":"Arts and Humanities"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10959","display_name":"Student Assessment and Feedback","score":0.1412000060081482,"subfield":{"id":"https://openalex.org/subfields/3304","display_name":"Education"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10467","display_name":"Psychometric Methodologies and Testing","score":0.06120000034570694,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/efl-writing-assessment","display_name":"EFL writing assessment","score":0.9509999752044678},{"id":"https://openalex.org/keywords/generalizability-theory","display_name":"generalizability theory","score":0.7929999828338623},{"id":"https://openalex.org/keywords/rater-effects","display_name":"rater effects","score":0.6520000100135803},{"id":"https://openalex.org/keywords/peer-ratings","display_name":"peer ratings","score":0.6620000004768372},{"id":"https://openalex.org/keywords/self-assessment","display_name":"self-assessment","score":0.7519999742507935},{"id":"https://openalex.org/keywords/scoring-methods","display_name":"scoring methods","score":0.5199999809265137},{"id":"https://openalex.org/keywords/integrated-writing-tasks","display_name":"integrated writing tasks","score":0.9399999976158142},{"id":"https://openalex.org/keywords/rater-reliability","display_name":"rater reliability","score":0.527999997138977},{"id":"https://openalex.org/keywords/score-validity","display_name":"score validity","score":0.5910000205039978},{"id":"https://openalex.org/keywords/number-of-raters","display_name":"number of raters","score":0.6919999718666077}],"concepts":[{"id":"https://openalex.org/C27158222","wikidata":"https://www.wikidata.org/wiki/Q5532422","display_name":"Generalizability theory","level":2,"score":0.8769353628158569},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.6909766793251038},{"id":"https://openalex.org/C196083921","wikidata":"https://www.wikidata.org/wiki/Q7915758","display_name":"Variance (accounting)","level":2,"score":0.6132530570030212},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.5850691795349121},{"id":"https://openalex.org/C3018076075","wikidata":"https://www.wikidata.org/wiki/Q1826427","display_name":"Variance components","level":2,"score":0.4705484211444855},{"id":"https://openalex.org/C61863361","wikidata":"https://www.wikidata.org/wiki/Q470749","display_name":"Inter-rater reliability","level":3,"score":0.46314096450805664},{"id":"https://openalex.org/C83849319","wikidata":"https://www.wikidata.org/wiki/Q7295720","display_name":"Rating scale","level":2,"score":0.39822885394096375},{"id":"https://openalex.org/C145420912","wikidata":"https://www.wikidata.org/wiki/Q853077","display_name":"Mathematics education","level":1,"score":0.3642650544643402},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.35545212030410767},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.3304307162761688},{"id":"https://openalex.org/C138496976","wikidata":"https://www.wikidata.org/wiki/Q175002","display_name":"Developmental psychology","level":1,"score":0.21029365062713623},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.15405520796775818},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C121955636","wikidata":"https://www.wikidata.org/wiki/Q4116214","display_name":"Accounting","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1186/s40468-021-00134-5","doi":"https://doi.org/10.1186/s40468-021-00134-5","is_oa":true,"landing_page_url":"https://doi.org/10.1186/s40468-021-00134-5","pdf_url":"https://languagetestingasia.springeropen.com/track/pdf/10.1186/s40468-021-00134-5","source":{"id":"https://openalex.org/S2736419388","display_name":"Language Testing in Asia","issn_l":"2229-0443","issn":["2229-0443"],"is_oa":true,"is_in_doaj":true,"is_core":true,"listed_in":["cwts-core","doaj","erih-plus","jufo-1","norway-1","poland-20","russia-white-list-1"],"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Language Testing in Asia","raw_type":"journal-article"},{"id":"pmh:oai:doaj.org/article:d3a9b55a30d24b82bd4f354aaa36c4c2","doi":null,"is_oa":true,"landing_page_url":"https://doaj.org/article/d3a9b55a30d24b82bd4f354aaa36c4c2","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Language Testing in Asia, Vol 11, Iss 1, Pp 1-27 (2021)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.1186/s40468-021-00134-5","doi":"https://doi.org/10.1186/s40468-021-00134-5","is_oa":true,"landing_page_url":"https://doi.org/10.1186/s40468-021-00134-5","pdf_url":"https://languagetestingasia.springeropen.com/track/pdf/10.1186/s40468-021-00134-5","source":{"id":"https://openalex.org/S2736419388","display_name":"Language Testing in Asia","issn_l":"2229-0443","issn":["2229-0443"],"is_oa":true,"is_in_doaj":true,"is_core":true,"listed_in":["cwts-core","doaj","erih-plus","jufo-1","norway-1","poland-20","russia-white-list-1"],"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Language Testing in Asia","raw_type":"journal-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","display_name":"Quality education","score":0.8278999924659729}],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3211915243.pdf","grobid_xml":"https://content.openalex.org/works/W3211915243.grobid-xml"},"referenced_works_count":89,"referenced_works":["https://openalex.org/W54189162","https://openalex.org/W141272839","https://openalex.org/W250445487","https://openalex.org/W335528204","https://openalex.org/W385765385","https://openalex.org/W410723623","https://openalex.org/W632122678","https://openalex.org/W653640245","https://openalex.org/W1413081681","https://openalex.org/W1515216811","https://openalex.org/W1527569809","https://openalex.org/W1580499675","https://openalex.org/W1825411771","https://openalex.org/W1835811054","https://openalex.org/W1958015814","https://openalex.org/W1963679057","https://openalex.org/W1973845803","https://openalex.org/W1984249886","https://openalex.org/W1992441585","https://openalex.org/W1995243899","https://openalex.org/W1996441519","https://openalex.org/W1998681598","https://openalex.org/W2006444775","https://openalex.org/W2013831266","https://openalex.org/W2015112080","https://openalex.org/W2018306678","https://openalex.org/W2019714422","https://openalex.org/W2021591245","https://openalex.org/W2023420063","https://openalex.org/W2028773126","https://openalex.org/W2031703645","https://openalex.org/W2036410403","https://openalex.org/W2042156288","https://openalex.org/W2042524001","https://openalex.org/W2042536471","https://openalex.org/W2043544836","https://openalex.org/W2047450090","https://openalex.org/W2049702257","https://openalex.org/W2053410856","https://openalex.org/W2062137446","https://openalex.org/W2063947667","https://openalex.org/W2077649418","https://openalex.org/W2087210518","https://openalex.org/W2124012504","https://openalex.org/W2126192123","https://openalex.org/W2128928244","https://openalex.org/W2130631175","https://openalex.org/W2134268013","https://openalex.org/W2135465946","https://openalex.org/W2154123060","https://openalex.org/W2156090605","https://openalex.org/W2156151441","https://openalex.org/W2157049701","https://openalex.org/W2159651246","https://openalex.org/W2162476713","https://openalex.org/W2166566383","https://openalex.org/W2167112690","https://openalex.org/W2182892159","https://openalex.org/W2246552111","https://openalex.org/W2251116010","https://openalex.org/W2259892487","https://openalex.org/W2283215851","https://openalex.org/W2319531873","https://openalex.org/W2337801273","https://openalex.org/W2490857485","https://openalex.org/W2553456540","https://openalex.org/W2578280343","https://openalex.org/W2599193626","https://openalex.org/W2736243464","https://openalex.org/W2902027532","https://openalex.org/W2905010601","https://openalex.org/W2916753089","https://openalex.org/W2961627078","https://openalex.org/W2970761611","https://openalex.org/W2980074621","https://openalex.org/W3182727064","https://openalex.org/W3192154063","https://openalex.org/W3197253206","https://openalex.org/W3216779397","https://openalex.org/W4238804190","https://openalex.org/W4285719527","https://openalex.org/W4293168481","https://openalex.org/W4392260510","https://openalex.org/W6621716021","https://openalex.org/W6640898641","https://openalex.org/W6991231953","https://openalex.org/W7014349428","https://openalex.org/W7042716477","https://openalex.org/W7058313776"],"related_works":["https://openalex.org/W2170081781","https://openalex.org/W4283384991","https://openalex.org/W2772752854","https://openalex.org/W4402702294","https://openalex.org/W1992530294","https://openalex.org/W2278505953","https://openalex.org/W2795326501","https://openalex.org/W3045076043","https://openalex.org/W1973490241","https://openalex.org/W2275620432"],"abstract_inverted_index":{"Abstract":[0],"The":[1,51,87,192],"present":[2],"study":[3],"attempted":[4],"to":[5,6,58,80,114,140,168,221,232,308],"investigate":[7,233],"factors":[8],"which":[9],"affect":[10],"EFL":[11],"writing":[12,34,84,260,271,322],"scores":[13],"through":[14],"using":[15],"generalizability":[16],"theory":[17],"(G-theory).":[18],"To":[19],"this":[20],"purpose,":[21],"one":[22,29,32,44],"hundred":[23],"and":[24,31,48,62,76,144,190,210,218,242,258,266,283,301,324],"twenty":[25],"students":[26,319],"participated":[27],"in":[28,105,130,155,171,205,278,313],"independent":[30],"integrated":[33,259],"tasks.":[35],"Proceeding,":[36],"their":[37,318,325],"performances":[38],"were":[39,197,230,262,281],"scored":[40],"by":[41,98,100,121,123,125,148,150,202,212,336],"six":[42],"raters:":[43],"self-rating,":[45],"three":[46,90],"peers,-rating":[47],"two":[49,321,333],"instructors-rating.":[50],"main":[52],"purpose":[53],"of":[54,65,74,78,83,93,102,107,127,132,138,152,157,163,184,194,207,214,236,244,269],"the":[55,60,81,96,115,119,141,146,164,169,182,185,200,222,234,263,270,310],"sudy":[56],"was":[57,178,187,285],"determine":[59],"relative":[61],"absolute":[63],"contributions":[64],"different":[66,237],"facets":[67,238],"such":[68],"as":[69,199,246],"student,":[70],"rater,":[71,240],"task,":[72,241],"method":[73,101,126,151],"scoring,":[75],"background":[77,106,131,156,206,213],"education":[79,215],"validity":[82],"assessment":[85,186],"scores.":[86],"results":[88],"indicated":[89,286],"major":[91],"sources":[92,193],"variance:":[94],"(a)":[95],"student":[97,120,147,201],"task":[99,124],"scoring":[103,128,153,245,256,334],"(nested":[104,129,154,204],"education)":[108,133,158,208],"interaction":[109,134,159],"(STM:B)":[110],"with":[111,136,161,216],"31.8%":[112],"contribution":[113,139,220,235],"total":[116,142],"variance,":[117,143,224],"(b)":[118],"rater":[122,149,203,211],"(SRTM:B)":[135],"26.5%":[137],"(c)":[145],"(SRM:B)":[160],"17.6%":[162],"contribution.":[165],"With":[166],"regard":[167],"G-coefficients":[170],"G-study":[172],"(relative":[173],"G-coefficient":[174,293],"\u2265":[175],"0.86),":[176],"it":[177,284],"also":[179],"found":[180],"that":[181,252,287],"result":[183],"highly":[188],"valid":[189,300],"reliable.":[191],"error":[195,223],"variance":[196],"detected":[198],"(SR:B)":[209],"99.2%":[217],"0.8%":[219],"respectively.":[225],"Additionally,":[226],"ten":[227],"separate":[228],"G-studies":[229],"conducted":[231,282],"across":[239],"methods":[243,335],"differentiation":[247],"facet.":[248],"These":[249],"studies":[250,276],"suggested":[251],"peer":[253],"rating,":[254],"analytical":[255],"method,":[257],"tasks":[261],"most":[264],"reliable":[265,302],"generalizable":[267],"designs":[268],"assessments.":[272],"Finally,":[273],"five":[274],"decision-making":[275],"(D-studies)":[277],"optimization":[279],"level":[280],"at":[288,331,337],"least":[289,332,338],"four":[290,339],"raters":[291],"(with":[292],"=":[294],"0.80)":[295],"are":[296],"necessary":[297],"for":[298],"a":[299],"assessment.":[303],"Based":[304],"on":[305,330],"these":[306],"results,":[307],"achieve":[309],"greatest":[311],"gain":[312],"generalizability,":[314],"teachers":[315],"should":[316,327],"have":[317],"take":[320],"assessments":[323],"performance":[326],"be":[328],"rated":[329],"raters.":[340]},"counts_by_year":[{"year":2026,"cited_by_count":7},{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":5},{"year":2023,"cited_by_count":1},{"year":2022,"cited_by_count":4}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2021-11-22T00:00:00"}