{"id":"https://openalex.org/W2554609727","doi":"https://doi.org/10.1080/0969594x.2016.1253542","title":"Validity of comparative judgement to assess academic writing: examining implications of its holistic character and building on a shared consensus","display_name":"Validity of comparative judgement to assess academic writing: examining implications of its holistic character and building on a shared consensus","publication_year":2016,"publication_date":"2016-11-17","ids":{"openalex":"https://openalex.org/W2554609727","doi":"https://doi.org/10.1080/0969594x.2016.1253542","mag":"2554609727"},"language":"en","primary_location":{"id":"doi:10.1080/0969594x.2016.1253542","doi":"https://doi.org/10.1080/0969594x.2016.1253542","is_oa":false,"landing_page_url":"https://doi.org/10.1080/0969594x.2016.1253542","pdf_url":null,"source":{"id":"https://openalex.org/S190677702","display_name":"Assessment in Education Principles Policy and Practice","issn_l":"0969-594X","issn":["0969-594X","1465-329X"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["anvur-class-a","anvur-scientific","cwts-core","erih-plus","jufo-2","ki-jl-2","norway-2","poland-200","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310319847","host_organization_name":"Routledge","host_organization_lineage":["https://openalex.org/P4310319847","https://openalex.org/P4310320547"],"host_organization_lineage_names":["Routledge","Taylor & Francis"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Assessment in Education: Principles, Policy & Practice","raw_type":"journal-article"},"volume":null,"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"http://hdl.handle.net/2078.1/184263","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5091225943","display_name":"Tine van Daal","orcid":"https://orcid.org/0000-0001-9398-9775","observed_orcids":["https://orcid.org/0000-0001-9398-9775"]},"institutions":[{"id":"https://openalex.org/I149213910","display_name":"University of Antwerp","ror":"https://ror.org/008x57b05","country_code":"BE","type":"education","lineage":["https://openalex.org/I149213910"]}],"countries":["BE"],"is_corresponding":true,"raw_author_name":"Tine van Daal","raw_affiliation_strings":["Faculty of Social Sciences, Department of Training and Education Sciences, University of Antwerp, Antwerp, Belgium"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Faculty of Social Sciences, Department of Training and Education Sciences, University of Antwerp, Antwerp, Belgium","institution_ids":["https://openalex.org/I149213910"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5087494627","display_name":"Marije Lesterhuis","orcid":"https://orcid.org/0000-0002-3808-556X","observed_orcids":["https://orcid.org/0000-0002-3808-556X"]},"institutions":[{"id":"https://openalex.org/I149213910","display_name":"University of Antwerp","ror":"https://ror.org/008x57b05","country_code":"BE","type":"education","lineage":["https://openalex.org/I149213910"]}],"countries":["BE"],"is_corresponding":false,"raw_author_name":"Marije Lesterhuis","raw_affiliation_strings":["Faculty of Social Sciences, Department of Training and Education Sciences, University of Antwerp, Antwerp, Belgium"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Faculty of Social Sciences, Department of Training and Education Sciences, University of Antwerp, Antwerp, Belgium","institution_ids":["https://openalex.org/I149213910"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5073092406","display_name":"Liesje Coertjens","orcid":"https://orcid.org/0000-0003-1209-5089","observed_orcids":["https://orcid.org/0000-0003-1209-5089"]},"institutions":[{"id":"https://openalex.org/I149213910","display_name":"University of Antwerp","ror":"https://ror.org/008x57b05","country_code":"BE","type":"education","lineage":["https://openalex.org/I149213910"]},{"id":"https://openalex.org/I95674353","display_name":"UCLouvain","ror":"https://ror.org/02495e989","country_code":"BE","type":"education","lineage":["https://openalex.org/I95674353"]}],"countries":["BE"],"is_corresponding":false,"raw_author_name":"Liesje Coertjens","raw_affiliation_strings":["Faculty of Social Sciences, Department of Training and Education Sciences, University of Antwerp, Antwerp, Belgium","Psychological Sciences Research Institute, Universit\u00e9 catholique de Louvain, Louvain-la-Neuve, Belgium"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Faculty of Social Sciences, Department of Training and Education Sciences, University of Antwerp, Antwerp, Belgium","institution_ids":["https://openalex.org/I149213910"]},{"raw_affiliation_string":"Psychological Sciences Research Institute, Universit\u00e9 catholique de Louvain, Louvain-la-Neuve, Belgium","institution_ids":["https://openalex.org/I95674353"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5006255224","display_name":"Vincent Donche","orcid":"https://orcid.org/0000-0002-9405-3896","observed_orcids":["https://orcid.org/0000-0002-9405-3896"]},"institutions":[{"id":"https://openalex.org/I149213910","display_name":"University of Antwerp","ror":"https://ror.org/008x57b05","country_code":"BE","type":"education","lineage":["https://openalex.org/I149213910"]}],"countries":["BE"],"is_corresponding":false,"raw_author_name":"Vincent Donche","raw_affiliation_strings":["Faculty of Social Sciences, Department of Training and Education Sciences, University of Antwerp, Antwerp, Belgium"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Faculty of Social Sciences, Department of Training and Education Sciences, University of Antwerp, Antwerp, Belgium","institution_ids":["https://openalex.org/I149213910"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5008069175","display_name":"Sven De M\u00e6yer","orcid":"https://orcid.org/0000-0003-2888-1631","observed_orcids":["https://orcid.org/0000-0003-2888-1631"]},"institutions":[{"id":"https://openalex.org/I149213910","display_name":"University of Antwerp","ror":"https://ror.org/008x57b05","country_code":"BE","type":"education","lineage":["https://openalex.org/I149213910"]}],"countries":["BE"],"is_corresponding":false,"raw_author_name":"Sven De Maeyer","raw_affiliation_strings":["Faculty of Social Sciences, Department of Training and Education Sciences, University of Antwerp, Antwerp, Belgium"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Faculty of Social Sciences, Department of Training and Education Sciences, University of Antwerp, Antwerp, Belgium","institution_ids":["https://openalex.org/I149213910"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":["https://openalex.org/A5091225943"],"corresponding_institution_ids":["https://openalex.org/I149213910"],"apc_list":null,"apc_paid":null,"fwci":22.6992,"has_fulltext":false,"cited_by_count":69,"citation_normalized_percentile":{"value":0.99261088,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":99},"biblio":{"volume":"26","issue":"1","first_page":"59","last_page":"74"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10959","display_name":"Student Assessment and Feedback","score":0.8931000232696533,"subfield":{"id":"https://openalex.org/subfields/3304","display_name":"Education"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10959","display_name":"Student Assessment and Feedback","score":0.8931000232696533,"subfield":{"id":"https://openalex.org/subfields/3304","display_name":"Education"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10467","display_name":"Psychometric Methodologies and Testing","score":0.026399999856948853,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T14025","display_name":"Educational Technology and Assessment","score":0.01360000018030405,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/comparative-judgment","display_name":"comparative judgment","score":0.9929999709129333},{"id":"https://openalex.org/keywords/essay-scoring","display_name":"essay scoring","score":0.7919999957084656},{"id":"https://openalex.org/keywords/academic-writing-assessment","display_name":"academic writing assessment","score":0.8460000157356262},{"id":"https://openalex.org/keywords/validity","display_name":"validity","score":0.7620000243186951},{"id":"https://openalex.org/keywords/holistic-assessment","display_name":"holistic assessment","score":0.6980000138282776},{"id":"https://openalex.org/keywords/writing-quality","display_name":"writing quality","score":0.47699999809265137},{"id":"https://openalex.org/keywords/competence-descriptions","display_name":"competence descriptions","score":0.5979999899864197},{"id":"https://openalex.org/keywords/rank-order","display_name":"rank order","score":0.3880000114440918},{"id":"https://openalex.org/keywords/inter-rater-reliability","display_name":"inter-rater reliability","score":0.42500001192092896}],"concepts":[{"id":"https://openalex.org/C2776548248","wikidata":"https://www.wikidata.org/wiki/Q12621536","display_name":"Judgement","level":2,"score":0.9005955457687378},{"id":"https://openalex.org/C2780871342","wikidata":"https://www.wikidata.org/wiki/Q7883752","display_name":"Underpinning","level":2,"score":0.7007946372032166},{"id":"https://openalex.org/C100521375","wikidata":"https://www.wikidata.org/wiki/Q2015382","display_name":"Competence (human resources)","level":2,"score":0.6844160556793213},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.5897650718688965},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.5213097929954529},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.4975567162036896},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.4000084698200226},{"id":"https://openalex.org/C147176958","wikidata":"https://www.wikidata.org/wiki/Q77590","display_name":"Civil engineering","level":1,"score":0.0},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1080/0969594x.2016.1253542","doi":"https://doi.org/10.1080/0969594x.2016.1253542","is_oa":false,"landing_page_url":"https://doi.org/10.1080/0969594x.2016.1253542","pdf_url":null,"source":{"id":"https://openalex.org/S190677702","display_name":"Assessment in Education Principles Policy and Practice","issn_l":"0969-594X","issn":["0969-594X","1465-329X"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["anvur-class-a","anvur-scientific","cwts-core","erih-plus","jufo-2","ki-jl-2","norway-2","poland-200","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310319847","host_organization_name":"Routledge","host_organization_lineage":["https://openalex.org/P4310319847","https://openalex.org/P4310320547"],"host_organization_lineage_names":["Routledge","Taylor & Francis"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Assessment in Education: Principles, Policy & Practice","raw_type":"journal-article"},{"id":"pmh:oai:dial.uclouvain.be:boreal:184263","doi":null,"is_oa":true,"landing_page_url":"http://hdl.handle.net/2078.1/184263","pdf_url":null,"source":{"id":"https://openalex.org/S4306401902","display_name":"Digital Access to Libraries (Universit\u00e9 catholique de Louvain (UCL), l'Universit\u00e9 de Namur (UNamur) and the Universit\u00e9 Saint-Louis (USL-B))","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I95674353","host_organization_name":"UCLouvain","host_organization_lineage":["https://openalex.org/I95674353"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Assessment in Education: Principles, Policy and Practice, Vol. 26, no. 1, p. 59-74 (2019)","raw_type":"info:eu-repo/semantics/article"}],"best_oa_location":{"id":"pmh:oai:dial.uclouvain.be:boreal:184263","doi":null,"is_oa":true,"landing_page_url":"http://hdl.handle.net/2078.1/184263","pdf_url":null,"source":{"id":"https://openalex.org/S4306401902","display_name":"Digital Access to Libraries (Universit\u00e9 catholique de Louvain (UCL), l'Universit\u00e9 de Namur (UNamur) and the Universit\u00e9 Saint-Louis (USL-B))","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I95674353","host_organization_name":"UCLouvain","host_organization_lineage":["https://openalex.org/I95674353"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Assessment in Education: Principles, Policy and Practice, Vol. 26, no. 1, p. 59-74 (2019)","raw_type":"info:eu-repo/semantics/article"},"sustainable_development_goals":[{"display_name":"Quality education","id":"https://metadata.un.org/sdg/4","score":0.6164000034332275}],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":39,"referenced_works":["https://openalex.org/W591603755","https://openalex.org/W605983222","https://openalex.org/W1533867010","https://openalex.org/W1555767810","https://openalex.org/W1719156708","https://openalex.org/W1918381019","https://openalex.org/W1970133260","https://openalex.org/W1979290264","https://openalex.org/W1984354219","https://openalex.org/W1984484958","https://openalex.org/W1985858262","https://openalex.org/W1991701363","https://openalex.org/W1992861205","https://openalex.org/W2003481038","https://openalex.org/W2009565870","https://openalex.org/W2010575178","https://openalex.org/W2012303852","https://openalex.org/W2013784666","https://openalex.org/W2028718514","https://openalex.org/W2033442452","https://openalex.org/W2064851486","https://openalex.org/W2067978114","https://openalex.org/W2102274116","https://openalex.org/W2106040789","https://openalex.org/W2107031757","https://openalex.org/W2144893176","https://openalex.org/W2145337364","https://openalex.org/W2146695179","https://openalex.org/W2150917105","https://openalex.org/W2165181820","https://openalex.org/W2165902651","https://openalex.org/W2170738300","https://openalex.org/W2171039963","https://openalex.org/W2592618247","https://openalex.org/W2658562839","https://openalex.org/W2755935388","https://openalex.org/W2899108003","https://openalex.org/W4242721106","https://openalex.org/W4300870773"],"related_works":["https://openalex.org/W4312133385","https://openalex.org/W4415254831","https://openalex.org/W7162857416","https://openalex.org/W4280593927","https://openalex.org/W4409205006","https://openalex.org/W4292411473","https://openalex.org/W4391775742","https://openalex.org/W2994443377","https://openalex.org/W7170082103","https://openalex.org/W4400782968"],"abstract_inverted_index":{"Recently,":[0],"comparative":[1,45],"judgement":[2],"has":[3],"been":[4],"introduced":[5],"as":[6],"an":[7],"alternative":[8],"method":[9,15],"for":[10,64],"scoring":[11],"essays.":[12,116],"Although":[13],"this":[14,141],"is":[16,29],"promising":[17],"in":[18,109,120],"terms":[19,121],"of":[20,44,83,115,122,146],"obtaining":[21],"reliable":[22],"scores,":[23],"empirical":[24],"evidence":[25],"concerning":[26],"its":[27,48],"validity":[28],"lacking.":[30],"The":[31,90],"current":[32],"study":[33,142],"examines":[34],"implications":[35],"resulting":[36],"from":[37],"two":[38],"critical":[39],"assumptions":[40],"underpinning":[41],"the":[42,53,58,81,100,113,130],"use":[43,106],"judgement,":[46],"namely:":[47],"holistic":[49],"characteristic":[50],"and":[51,128],"how":[52,123],"final":[54],"rank":[55],"order":[56,110],"reflects":[57],"shared":[59],"consensus":[60],"on":[61],"what":[62],"makes":[63],"a":[65],"good":[66],"essay.":[67],"Judges\u2019":[68],"justifications":[69],"that":[70,93],"underpin":[71],"their":[72,107,137],"decisions":[73],"are":[74,96],"qualitatively":[75],"analysed":[76],"to":[77,99,111,132],"obtain":[78],"insight":[79],"into":[80,88,136],"dimensions":[82],"academic":[84],"writing":[85,126],"they":[86,124,134],"take":[87],"account.":[89],"results":[91],"show":[92],"most":[94],"arguments":[95],"directly":[97],"related":[98],"competence":[101],"description.":[102],"However,":[103],"judges":[104,118],"also":[105],"expertise":[108],"judge":[112],"quality":[114],"Additionally,":[117],"differ":[119],"conceptualise":[125],"quality,":[127],"regarding":[129],"extent":[131],"which":[133],"tap":[135],"own":[138],"expertise.":[139],"Finally,":[140],"explores":[143],"diverging":[144],"conceptualisation":[145],"misfitting":[147],"judges.":[148]},"counts_by_year":[{"year":2026,"cited_by_count":5},{"year":2025,"cited_by_count":6},{"year":2024,"cited_by_count":7},{"year":2023,"cited_by_count":5},{"year":2022,"cited_by_count":16},{"year":2021,"cited_by_count":8},{"year":2020,"cited_by_count":5},{"year":2019,"cited_by_count":10},{"year":2018,"cited_by_count":5},{"year":2017,"cited_by_count":2}],"updated_date":"2026-11-04T10:54:05.112534","created_date":"2016-11-30T00:00:00"}