{"id":"https://openalex.org/W2944864183","doi":"https://doi.org/10.1111/emip.12329","title":"Rubric Rating with MFRM versus Randomly Distributed Comparative Judgment: A Comparison of Two Approaches to Second\u2010Language Writing Assessment","display_name":"Rubric Rating with MFRM versus Randomly Distributed Comparative Judgment: A Comparison of Two Approaches to Second\u2010Language Writing Assessment","publication_year":2020,"publication_date":"2020-05-01","ids":{"openalex":"https://openalex.org/W2944864183","doi":"https://doi.org/10.1111/emip.12329","mag":"2944864183"},"language":"en","primary_location":{"id":"doi:10.1111/emip.12329","doi":"https://doi.org/10.1111/emip.12329","is_oa":false,"landing_page_url":"https://doi.org/10.1111/emip.12329","pdf_url":null,"source":{"id":"https://openalex.org/S82984449","display_name":"Educational Measurement Issues and Practice","issn_l":"0731-1745","issn":["0731-1745","1745-3992"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["anvur-class-a","anvur-scientific","cwts-core","dongbi-b","jufo-1","norway-1","poland-140","russia-white-list-2"],"host_organization":"https://openalex.org/P4310320595","host_organization_name":"Wiley","host_organization_lineage":["https://openalex.org/P4310320595"],"host_organization_lineage_names":["Wiley"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Educational Measurement: Issues and Practice","raw_type":"journal-article"},"volume":null,"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Maureen E. Sims","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I100005738","display_name":"Brigham Young University","ror":"https://ror.org/047rhhm47","country_code":"US","type":"education","lineage":["https://openalex.org/I100005738"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Maureen E. Sims","raw_affiliation_strings":["Brigham Young University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Brigham Young University","institution_ids":["https://openalex.org/I100005738"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5020084420","display_name":"Troy Leigh Cox","orcid":"https://orcid.org/0000-0001-9379-5102","observed_orcids":["https://orcid.org/0000-0001-9379-5102"]},"institutions":[{"id":"https://openalex.org/I100005738","display_name":"Brigham Young University","ror":"https://ror.org/047rhhm47","country_code":"US","type":"education","lineage":["https://openalex.org/I100005738"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Troy L. Cox","raw_affiliation_strings":["Brigham Young University"],"raw_orcid":"https://orcid.org/0000-0001-9379-5102","affiliations":[{"raw_affiliation_string":"Brigham Young University","institution_ids":["https://openalex.org/I100005738"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5033329401","display_name":"Grant Eckstein","orcid":"https://orcid.org/0000-0002-3667-4571","observed_orcids":["https://orcid.org/0000-0002-3667-4571"]},"institutions":[{"id":"https://openalex.org/I100005738","display_name":"Brigham Young University","ror":"https://ror.org/047rhhm47","country_code":"US","type":"education","lineage":["https://openalex.org/I100005738"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Grant T. Eckstein","raw_affiliation_strings":["Brigham Young University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Brigham Young University","institution_ids":["https://openalex.org/I100005738"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5044026676","display_name":"K. James Hartshorn","orcid":"https://orcid.org/0000-0002-0629-7410","observed_orcids":["https://orcid.org/0000-0002-0629-7410"]},"institutions":[{"id":"https://openalex.org/I100005738","display_name":"Brigham Young University","ror":"https://ror.org/047rhhm47","country_code":"US","type":"education","lineage":["https://openalex.org/I100005738"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"K. James Hartshorn","raw_affiliation_strings":["Brigham Young University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Brigham Young University","institution_ids":["https://openalex.org/I100005738"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5085417597","display_name":"Matthew Porter Wilcox","orcid":"https://orcid.org/0000-0002-6020-529X","observed_orcids":["https://orcid.org/0000-0002-6020-529X"]},"institutions":[{"id":"https://openalex.org/I100005738","display_name":"Brigham Young University","ror":"https://ror.org/047rhhm47","country_code":"US","type":"education","lineage":["https://openalex.org/I100005738"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Matthew P. Wilcox","raw_affiliation_strings":["Brigham Young University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Brigham Young University","institution_ids":["https://openalex.org/I100005738"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5112277550","display_name":"Judson M. Hart","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I100005738","display_name":"Brigham Young University","ror":"https://ror.org/047rhhm47","country_code":"US","type":"education","lineage":["https://openalex.org/I100005738"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Judson M. Hart","raw_affiliation_strings":["Brigham Young University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Brigham Young University","institution_ids":["https://openalex.org/I100005738"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I100005738"],"apc_list":{"value":3240,"currency":"USD","value_usd":3240},"apc_paid":null,"fwci":0.9543,"has_fulltext":false,"cited_by_count":34,"citation_normalized_percentile":{"value":0.76527988,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":88,"max":100},"biblio":{"volume":"39","issue":"4","first_page":"30","last_page":"40"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10467","display_name":"Psychometric Methodologies and Testing","score":0.7318999767303467,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10467","display_name":"Psychometric Methodologies and Testing","score":0.7318999767303467,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10021","display_name":"EFL/ESL Teaching and Learning","score":0.12790000438690186,"subfield":{"id":"https://openalex.org/subfields/1203","display_name":"Language and Linguistics"},"field":{"id":"https://openalex.org/fields/12","display_name":"Arts and Humanities"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10959","display_name":"Student Assessment and Feedback","score":0.10140000283718109,"subfield":{"id":"https://openalex.org/subfields/3304","display_name":"Education"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/second-language-writing-assessment","display_name":"second language writing assessment","score":0.8009999990463257},{"id":"https://openalex.org/keywords/esl-writing","display_name":"ESL writing","score":0.890999972820282},{"id":"https://openalex.org/keywords/many-facet-rasch-measurement","display_name":"many-facet Rasch model","score":0.8790000081062317},{"id":"https://openalex.org/keywords/comparative-judgment","display_name":"comparative judgment","score":0.9020000100135803},{"id":"https://openalex.org/keywords/rater-reliability","display_name":"rater reliability","score":0.8259999752044678},{"id":"https://openalex.org/keywords/essay-scoring","display_name":"essay scoring","score":0.597000002861023}],"concepts":[{"id":"https://openalex.org/C111640148","wikidata":"https://www.wikidata.org/wiki/Q847349","display_name":"Rubric","level":2,"score":0.9767159819602966},{"id":"https://openalex.org/C2777898490","wikidata":"https://www.wikidata.org/wiki/Q17157236","display_name":"Writing assessment","level":2,"score":0.6721356511116028},{"id":"https://openalex.org/C101266164","wikidata":"https://www.wikidata.org/wiki/Q2131821","display_name":"Rasch model","level":2,"score":0.6510253548622131},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.617032527923584},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.6106131076812744},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5833473205566406},{"id":"https://openalex.org/C173633133","wikidata":"https://www.wikidata.org/wiki/Q2110223","display_name":"Raw score","level":3,"score":0.5270942449569702},{"id":"https://openalex.org/C61863361","wikidata":"https://www.wikidata.org/wiki/Q470749","display_name":"Inter-rater reliability","level":3,"score":0.48360103368759155},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.4717871844768524},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3935944139957428},{"id":"https://openalex.org/C83849319","wikidata":"https://www.wikidata.org/wiki/Q7295720","display_name":"Rating scale","level":2,"score":0.3558852970600128},{"id":"https://openalex.org/C75630572","wikidata":"https://www.wikidata.org/wiki/Q538904","display_name":"Applied psychology","level":1,"score":0.3517012894153595},{"id":"https://openalex.org/C145420912","wikidata":"https://www.wikidata.org/wiki/Q853077","display_name":"Mathematics education","level":1,"score":0.29326000809669495},{"id":"https://openalex.org/C132964779","wikidata":"https://www.wikidata.org/wiki/Q2110223","display_name":"Raw data","level":2,"score":0.13968127965927124},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C163258240","wikidata":"https://www.wikidata.org/wiki/Q25342","display_name":"Power (physics)","level":2,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C138496976","wikidata":"https://www.wikidata.org/wiki/Q175002","display_name":"Developmental psychology","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1111/emip.12329","doi":"https://doi.org/10.1111/emip.12329","is_oa":false,"landing_page_url":"https://doi.org/10.1111/emip.12329","pdf_url":null,"source":{"id":"https://openalex.org/S82984449","display_name":"Educational Measurement Issues and Practice","issn_l":"0731-1745","issn":["0731-1745","1745-3992"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["anvur-class-a","anvur-scientific","cwts-core","dongbi-b","jufo-1","norway-1","poland-140","russia-white-list-2"],"host_organization":"https://openalex.org/P4310320595","host_organization_name":"Wiley","host_organization_lineage":["https://openalex.org/P4310320595"],"host_organization_lineage_names":["Wiley"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Educational Measurement: Issues and Practice","raw_type":"journal-article"},{"id":"pmh:oai:scholarsarchive.byu.edu:facpub-8952","doi":null,"is_oa":false,"landing_page_url":"https://scholarsarchive.byu.edu/facpub/8031","pdf_url":null,"source":{"id":"https://openalex.org/S4377196308","display_name":"ScholarsArchive  (Brigham Young University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I100005738","host_organization_name":"Brigham Young University","host_organization_lineage":["https://openalex.org/I100005738"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Faculty Publications","raw_type":"text"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.5385000109672546,"id":"https://metadata.un.org/sdg/4","display_name":"Quality education"}],"sustainable_development_goals_aurora":[{"score":0.8700000047683716,"id":"https://metadata.un.org/sdg/4","display_name":"Quality education"}],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":40,"referenced_works":["https://openalex.org/W18399107","https://openalex.org/W41392895","https://openalex.org/W104765354","https://openalex.org/W165891335","https://openalex.org/W345157124","https://openalex.org/W632980047","https://openalex.org/W637436370","https://openalex.org/W847524857","https://openalex.org/W1054264357","https://openalex.org/W1555767810","https://openalex.org/W1570325522","https://openalex.org/W1704942559","https://openalex.org/W1970931621","https://openalex.org/W1984354219","https://openalex.org/W1985858262","https://openalex.org/W2006444775","https://openalex.org/W2010575178","https://openalex.org/W2018532708","https://openalex.org/W2028718514","https://openalex.org/W2028773126","https://openalex.org/W2028829385","https://openalex.org/W2029471204","https://openalex.org/W2033442452","https://openalex.org/W2063827521","https://openalex.org/W2073084141","https://openalex.org/W2094218279","https://openalex.org/W2115389369","https://openalex.org/W2119479814","https://openalex.org/W2119844339","https://openalex.org/W2128896901","https://openalex.org/W2145770094","https://openalex.org/W2146695179","https://openalex.org/W2165181820","https://openalex.org/W2179650432","https://openalex.org/W2316790024","https://openalex.org/W2493796278","https://openalex.org/W2623875134","https://openalex.org/W2864259650","https://openalex.org/W3202972339","https://openalex.org/W3204408364"],"related_works":["https://openalex.org/W7043001769","https://openalex.org/W4411629469","https://openalex.org/W4226096478","https://openalex.org/W4285740697","https://openalex.org/W4313302793","https://openalex.org/W2028773126","https://openalex.org/W2182892159","https://openalex.org/W2739065034","https://openalex.org/W2099062016","https://openalex.org/W2981173205"],"abstract_inverted_index":{"Abstract":[0],"The":[1],"purpose":[2],"of":[3,11,24,48,70,76,99,117],"this":[4,165],"study":[5,42,166],"is":[6,31,151],"to":[7,17,57,154,186],"explore":[8],"the":[9,22,44,61,97,100,106,126],"reliability":[10],"a":[12,32,114,145,181],"potentially":[13],"more":[14,146],"practical":[15,147],"approach":[16],"direct":[18],"writing":[19,50],"assessment":[20,51,156],"in":[21,68,179],"context":[23],"ESL":[25,49,177],"writing.":[26],"Traditional":[27],"rubric":[28,46],"rating":[29,182],"(RR)":[30],"common":[33],"yet":[34],"resource\u2010intensive":[35],"evaluation":[36],"practice":[37],"when":[38],"performed":[39],"reliably.":[40],"This":[41],"compared":[43,96],"traditional":[45],"model":[47,183],"and":[52,78,90,102,113],"many\u2010facet":[53],"Rasch":[54],"modeling":[55],"(MFRM)":[56],"comparative":[58,120],"judgment":[59,121],"(CJ),":[60],"new":[62],"approach,":[63,128],"which":[64],"shows":[65],"promising":[66],"results":[67,98],"terms":[69],"reliability.":[71],"We":[72,95],"employed":[73],"two":[74],"groups":[75,104],"raters\u2014novice":[77],"experienced\u2014and":[79],"used":[80],"essays":[81],"that":[82,125],"had":[83],"been":[84],"previously":[85],"double\u2010rated,":[86],"analyzed":[87],"with":[88,92],"MFRM,":[89,112],"selected":[91],"fit":[93],"statistics.":[94],"novice":[101],"experienced":[103],"against":[105],"initial":[107],"ratings":[108],"using":[109],"raw":[110],"scores,":[111],"modern":[115],"form":[116],"CJ\u2014randomly":[118],"distributed":[119],"(RDCJ).":[122],"Results":[123,163],"showed":[124],"CJ":[127,150],"though":[129],"not":[130,168],"appropriate":[131],"for":[132],"all":[133],"contexts,":[134],"can":[135,174],"be":[136],"as":[137,139,144],"reliable":[138],"RR":[140],"while":[141,158],"showing":[142],"promise":[143],"approach.":[148],"Additionally,":[149],"easily":[152],"transferable":[153],"novel":[155],"tasks":[157],"still":[159],"providing":[160],"context\u2010specific":[161],"scores.":[162],"from":[164],"will":[167],"only":[169],"inform":[170],"future":[171],"studies":[172],"but":[173],"help":[175],"guide":[176],"programs":[178],"selecting":[180],"best":[184],"suited":[185],"their":[187],"specific":[188],"needs.":[189]},"counts_by_year":[{"year":2025,"cited_by_count":5},{"year":2024,"cited_by_count":22},{"year":2023,"cited_by_count":2},{"year":2022,"cited_by_count":3},{"year":2021,"cited_by_count":1},{"year":2013,"cited_by_count":1}],"updated_date":"2026-10-08T09:06:37.294493","created_date":"2019-05-29T00:00:00"}