{"id":"https://openalex.org/W4312294782","doi":"https://doi.org/10.1145/3551708.3556204","title":"Using Elo Rating as a Metric for Comparative Judgement in Educational Assessment","display_name":"Using Elo Rating as a Metric for Comparative Judgement in Educational Assessment","publication_year":2022,"publication_date":"2022-07-13","ids":{"openalex":"https://openalex.org/W4312294782","doi":"https://doi.org/10.1145/3551708.3556204"},"language":"en","primary_location":{"id":"doi:10.1145/3551708.3556204","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3551708.3556204","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 6th International Conference on Education and Multimedia Technology","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5065882234","display_name":"Andy Gray","orcid":"https://orcid.org/0000-0002-1150-2052"},"institutions":[{"id":"https://openalex.org/I39586589","display_name":"Swansea University","ror":"https://ror.org/053fq8t95","country_code":"GB","type":"education","lineage":["https://openalex.org/I39586589"]}],"countries":["GB"],"is_corresponding":true,"raw_author_name":"Andy Gray","raw_affiliation_strings":["Department of Computer Science, Swansea University, UK"],"affiliations":[{"raw_affiliation_string":"Department of Computer Science, Swansea University, UK","institution_ids":["https://openalex.org/I39586589"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5023675295","display_name":"Alma Rahat","orcid":"https://orcid.org/0000-0002-5023-1371"},"institutions":[{"id":"https://openalex.org/I39586589","display_name":"Swansea University","ror":"https://ror.org/053fq8t95","country_code":"GB","type":"education","lineage":["https://openalex.org/I39586589"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Alma A.M Rahat","raw_affiliation_strings":["Department of Computer Science, Swansea University, UK"],"affiliations":[{"raw_affiliation_string":"Department of Computer Science, Swansea University, UK","institution_ids":["https://openalex.org/I39586589"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5015452463","display_name":"Tom Crick","orcid":"https://orcid.org/0000-0001-5196-9389"},"institutions":[{"id":"https://openalex.org/I39586589","display_name":"Swansea University","ror":"https://ror.org/053fq8t95","country_code":"GB","type":"education","lineage":["https://openalex.org/I39586589"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Tom Crick","raw_affiliation_strings":["Faculty of Humanities & Social Sciences & Department of Computer Science, Swansea University, UK"],"affiliations":[{"raw_affiliation_string":"Faculty of Humanities & Social Sciences & Department of Computer Science, Swansea University, UK","institution_ids":["https://openalex.org/I39586589"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5069810884","display_name":"Stephen Lindsay","orcid":"https://orcid.org/0000-0001-6063-3676"},"institutions":[{"id":"https://openalex.org/I7882870","display_name":"University of Glasgow","ror":"https://ror.org/00vtgdb53","country_code":"GB","type":"education","lineage":["https://openalex.org/I7882870"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Stephen Lindsay","raw_affiliation_strings":["School of Computer Science, Glasgow University, UK"],"affiliations":[{"raw_affiliation_string":"School of Computer Science, Glasgow University, UK","institution_ids":["https://openalex.org/I7882870"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5102447208","display_name":"Darren Wallace","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Darren Wallace","raw_affiliation_strings":["CDSM, UK"],"affiliations":[{"raw_affiliation_string":"CDSM, UK","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5065882234"],"corresponding_institution_ids":["https://openalex.org/I39586589"],"apc_list":null,"apc_paid":null,"fwci":1.7794,"has_fulltext":false,"cited_by_count":3,"citation_normalized_percentile":{"value":0.8801336,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":96},"biblio":{"volume":null,"issue":null,"first_page":"272","last_page":"278"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10959","display_name":"Student Assessment and Feedback","score":0.9962000250816345,"subfield":{"id":"https://openalex.org/subfields/3304","display_name":"Education"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10959","display_name":"Student Assessment and Feedback","score":0.9962000250816345,"subfield":{"id":"https://openalex.org/subfields/3304","display_name":"Education"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T13405","display_name":"Educational Assessment and Improvement","score":0.9883999824523926,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12010","display_name":"Evaluation and Performance Assessment","score":0.98089998960495,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/pairwise-comparison","display_name":"Pairwise comparison","score":0.801802933216095},{"id":"https://openalex.org/keywords/judgement","display_name":"Judgement","score":0.7791118621826172},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6634008884429932},{"id":"https://openalex.org/keywords/metric","display_name":"Metric (unit)","score":0.5305212736129761},{"id":"https://openalex.org/keywords/rank","display_name":"Rank (graph theory)","score":0.5125872492790222},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.4775581955909729},{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.42428550124168396},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.415809690952301},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.33160847425460815},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.3312203288078308},{"id":"https://openalex.org/keywords/mathematics-education","display_name":"Mathematics education","score":0.32040420174598694},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.31874459981918335},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.15284892916679382},{"id":"https://openalex.org/keywords/law","display_name":"Law","score":0.0865597128868103},{"id":"https://openalex.org/keywords/political-science","display_name":"Political science","score":0.08389922976493835}],"concepts":[{"id":"https://openalex.org/C184898388","wikidata":"https://www.wikidata.org/wiki/Q1435712","display_name":"Pairwise comparison","level":2,"score":0.801802933216095},{"id":"https://openalex.org/C2776548248","wikidata":"https://www.wikidata.org/wiki/Q12621536","display_name":"Judgement","level":2,"score":0.7791118621826172},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6634008884429932},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.5305212736129761},{"id":"https://openalex.org/C164226766","wikidata":"https://www.wikidata.org/wiki/Q7293202","display_name":"Rank (graph theory)","level":2,"score":0.5125872492790222},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.4775581955909729},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.42428550124168396},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.415809690952301},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.33160847425460815},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.3312203288078308},{"id":"https://openalex.org/C145420912","wikidata":"https://www.wikidata.org/wiki/Q853077","display_name":"Mathematics education","level":1,"score":0.32040420174598694},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.31874459981918335},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.15284892916679382},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0865597128868103},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.08389922976493835},{"id":"https://openalex.org/C21547014","wikidata":"https://www.wikidata.org/wiki/Q1423657","display_name":"Operations management","level":1,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C163258240","wikidata":"https://www.wikidata.org/wiki/Q25342","display_name":"Power (physics)","level":2,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C114614502","wikidata":"https://www.wikidata.org/wiki/Q76592","display_name":"Combinatorics","level":1,"score":0.0},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1145/3551708.3556204","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3551708.3556204","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 6th International Conference on Education and Multimedia Technology","raw_type":"proceedings-article"},{"id":"pmh:oai:research-information.bris.ac.uk:openaire_cris_publications/a95ac49d-2270-4311-8f22-bd30ed1e0629","is_oa":false,"landing_page_url":"https://research-information.bris.ac.uk/en/publications/a95ac49d-2270-4311-8f22-bd30ed1e0629","pdf_url":null,"source":{"id":"https://openalex.org/S4306400895","display_name":"Bristol Research (University of Bristol)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I36234482","host_organization_name":"University of Bristol","host_organization_lineage":["https://openalex.org/I36234482"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Gray, A, Rahat, A A M, Crick, T, Lindsay, S & Wallace, D 2022, Using Elo Rating as a Metric for Comparative Judgement in Educational Assessment. in ICEMT '22 : Proceedings of the 6th International Conference on Education and Multimedia Technology. Proceedings of the International Conference on Education and Multimedia Technology, Association for Computing Machinery, pp. 272-278. https://doi.org/10.1145/3551708.3556204","raw_type":"contributionToPeriodical"},{"id":"pmh:oai:researchspace.bathspa.ac.uk:16280","is_oa":false,"landing_page_url":"http://researchspace.bathspa.ac.uk/16280/","pdf_url":null,"source":{"id":"https://openalex.org/S4306401821","display_name":"ResearchSPAce (Bath Spa University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I91047523","host_organization_name":"Bath Spa University","host_organization_lineage":["https://openalex.org/I91047523"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Book Chapter or Section"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.5400000214576721,"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4"}],"awards":[{"id":"https://openalex.org/G7216120360","display_name":"EPSRC Centre for Doctoral Training in Enhancing Human Interactions and Collaborations with Data and Intelligence Driven Systems","funder_award_id":"EP/S021892/1","funder_id":"https://openalex.org/F4320334627","funder_display_name":"Engineering and Physical Sciences Research Council"}],"funders":[{"id":"https://openalex.org/F4320334627","display_name":"Engineering and Physical Sciences Research Council","ror":"https://ror.org/0439y7842"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":17,"referenced_works":["https://openalex.org/W576567619","https://openalex.org/W645009165","https://openalex.org/W1564486018","https://openalex.org/W2012303852","https://openalex.org/W2125484256","https://openalex.org/W2293494073","https://openalex.org/W2335612402","https://openalex.org/W2592618247","https://openalex.org/W2789511283","https://openalex.org/W2994443377","https://openalex.org/W3015795228","https://openalex.org/W3033300489","https://openalex.org/W3061995964","https://openalex.org/W3133331818","https://openalex.org/W3161674196","https://openalex.org/W3174462591","https://openalex.org/W4205544055"],"related_works":["https://openalex.org/W2886802431","https://openalex.org/W4387575966","https://openalex.org/W2384262901","https://openalex.org/W2975214487","https://openalex.org/W651304006","https://openalex.org/W2345720417","https://openalex.org/W2385564112","https://openalex.org/W2324362495","https://openalex.org/W4236616503","https://openalex.org/W2894446834"],"abstract_inverted_index":{"Marking":[0],"and":[1,8,17,28,35,58,131],"feedback":[2,39],"are":[3],"essential":[4],"features":[5],"of":[6,14,26,81,191],"teaching":[7],"learning,":[9],"across":[10],"the":[11,41,52,74,79,91,138,148,154,157,180],"overwhelming":[12],"majority":[13],"educational":[15,75],"settings":[16],"contexts.":[18],"However,":[19],"it":[20,44,135],"can":[21,116,150,177],"take":[22],"a":[23,47,95,101,187],"great":[24],"deal":[25],"time":[27],"effort":[29],"for":[30,110,137,167],"teachers":[31],"to":[32,36,40,64,120,164],"mark":[33],"assessments,":[34],"provide":[37],"useful":[38],"students.":[42],"Furthermore,":[43],"also":[45,173],"creates":[46],"significant":[48],"cognitive":[49],"load":[50],"on":[51],"assessors,":[53],"especially":[54],"in":[55,73,153,179],"ensuring":[56],"fairness":[57],"equity.":[59],"Therefore,":[60],"an":[61,165],"alternative":[62],"approach":[63],"marking":[65,182,192],"called":[66],"comparative":[67,82],"judgement":[68],"(CJ)":[69],"has":[70],"been":[71],"proposed":[72],"space.":[76],"Inspired":[77],"by":[78,100],"law":[80],"judgment":[83],"(LCJ).":[84],"The":[85],"key":[86],"idea":[87],"here":[88],"is":[89,128,160],"that":[90,126,147,175],"better":[92],"submission":[93,159],"between":[94],"pair":[96],"will":[97],"be":[98,118,199],"identified":[99],"suitably":[102],"qualified":[103],"or":[104],"experienced":[105],"assessor.":[106],"This":[107],"pairwise":[108],"comparison":[109],"as":[111,114,156,193],"many":[112,162,196],"pairs":[113,197],"possible":[115],"then":[117],"used":[119],"rank":[121],"all":[122],"submissions.":[123],"Studies":[124],"suggest":[125],"CJ":[127,176],"highly":[129],"reliable":[130],"accurate":[132],"while":[133],"making":[134],"quick":[136],"teachers.":[139],"Alternative":[140],"studies":[141,171],"have":[142,172],"questioned":[143],"this":[144],"claim":[145],"suggesting":[146],"process":[149,183],"increase":[151],"bias":[152],"results":[155],"same":[158],"shown":[161],"times":[163],"assessor":[166],"increasing":[168],"reliability.":[169],"Additionally,":[170],"found":[174],"result":[178],"overall":[181],"taking":[184],"longer":[185],"than":[186],"more":[188],"traditional":[189],"method":[190],"information":[194],"about":[195],"must":[198],"collected.":[200]},"counts_by_year":[{"year":2023,"cited_by_count":2},{"year":2022,"cited_by_count":1}],"updated_date":"2026-03-18T14:38:29.013473","created_date":"2025-10-10T00:00:00"}