{"id":"https://openalex.org/W7108483237","doi":"https://doi.org/10.3390/computers14120530","title":"EvalCouncil: A Committee-Based LLM Framework for Reliable and Unbiased Automated Grading","display_name":"EvalCouncil: A Committee-Based LLM Framework for Reliable and Unbiased Automated Grading","publication_year":2025,"publication_date":"2025-12-03","ids":{"openalex":"https://openalex.org/W7108483237","doi":"https://doi.org/10.3390/computers14120530"},"language":"en","primary_location":{"id":"doi:10.3390/computers14120530","is_oa":true,"landing_page_url":"https://doi.org/10.3390/computers14120530","pdf_url":null,"source":{"id":"https://openalex.org/S4210228075","display_name":"Computers","issn_l":"2073-431X","issn":["2073-431X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310310987","host_organization_name":"Multidisciplinary Digital Publishing Institute","host_organization_lineage":["https://openalex.org/P4310310987"],"host_organization_lineage_names":["Multidisciplinary Digital Publishing Institute"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Computers","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.3390/computers14120530","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Catalin Anghel","orcid":"https://orcid.org/0000-0002-1849-3072"},"institutions":[{"id":"https://openalex.org/I3122748864","display_name":"\"Dunarea de Jos\" University of Galati","ror":"https://ror.org/052sta926","country_code":"RO","type":"education","lineage":["https://openalex.org/I3122748864"]}],"countries":["RO"],"is_corresponding":true,"raw_author_name":"Catalin Anghel","raw_affiliation_strings":["Department of Computer Science and Information Technology, \u201cDun\u0103rea de Jos\u201d University of Galati, \u0218tiin\u021bei St. 2, 800146 Galati, Romania"],"affiliations":[{"raw_affiliation_string":"Department of Computer Science and Information Technology, \u201cDun\u0103rea de Jos\u201d University of Galati, \u0218tiin\u021bei St. 2, 800146 Galati, Romania","institution_ids":["https://openalex.org/I3122748864"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Marian Viorel Craciun","orcid":"https://orcid.org/0009-0006-9970-2556"},"institutions":[{"id":"https://openalex.org/I3122748864","display_name":"\"Dunarea de Jos\" University of Galati","ror":"https://ror.org/052sta926","country_code":"RO","type":"education","lineage":["https://openalex.org/I3122748864"]}],"countries":["RO"],"is_corresponding":false,"raw_author_name":"Marian Viorel Craciun","raw_affiliation_strings":["Department of Computer Science and Information Technology, \u201cDun\u0103rea de Jos\u201d University of Galati, \u0218tiin\u021bei St. 2, 800146 Galati, Romania"],"affiliations":[{"raw_affiliation_string":"Department of Computer Science and Information Technology, \u201cDun\u0103rea de Jos\u201d University of Galati, \u0218tiin\u021bei St. 2, 800146 Galati, Romania","institution_ids":["https://openalex.org/I3122748864"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Andreea Alexandra Anghel","orcid":"https://orcid.org/0009-0008-2537-6713"},"institutions":[{"id":"https://openalex.org/I3122748864","display_name":"\"Dunarea de Jos\" University of Galati","ror":"https://ror.org/052sta926","country_code":"RO","type":"education","lineage":["https://openalex.org/I3122748864"]}],"countries":["RO"],"is_corresponding":false,"raw_author_name":"Andreea Alexandra Anghel","raw_affiliation_strings":["Faculty of Automation, Computer Science, Electrical and Electronic Engineering, \u201cDun\u0103rea de Jos\u201d University of Galati, 800008 Galati, Romania"],"affiliations":[{"raw_affiliation_string":"Faculty of Automation, Computer Science, Electrical and Electronic Engineering, \u201cDun\u0103rea de Jos\u201d University of Galati, 800008 Galati, Romania","institution_ids":["https://openalex.org/I3122748864"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Adina Cocu","orcid":"https://orcid.org/0000-0003-0935-4713"},"institutions":[{"id":"https://openalex.org/I3122748864","display_name":"\"Dunarea de Jos\" University of Galati","ror":"https://ror.org/052sta926","country_code":"RO","type":"education","lineage":["https://openalex.org/I3122748864"]}],"countries":["RO"],"is_corresponding":false,"raw_author_name":"Adina Cocu","raw_affiliation_strings":["Department of Computer Science and Information Technology, \u201cDun\u0103rea de Jos\u201d University of Galati, \u0218tiin\u021bei St. 2, 800146 Galati, Romania"],"affiliations":[{"raw_affiliation_string":"Department of Computer Science and Information Technology, \u201cDun\u0103rea de Jos\u201d University of Galati, \u0218tiin\u021bei St. 2, 800146 Galati, Romania","institution_ids":["https://openalex.org/I3122748864"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Antonio Stefan Balau","orcid":"https://orcid.org/0009-0002-5261-2060"},"institutions":[{"id":"https://openalex.org/I3122748864","display_name":"\"Dunarea de Jos\" University of Galati","ror":"https://ror.org/052sta926","country_code":"RO","type":"education","lineage":["https://openalex.org/I3122748864"]}],"countries":["RO"],"is_corresponding":false,"raw_author_name":"Antonio Stefan Balau","raw_affiliation_strings":["Faculty of Automation, Computer Science, Electrical and Electronic Engineering, \u201cDun\u0103rea de Jos\u201d University of Galati, 800008 Galati, Romania"],"affiliations":[{"raw_affiliation_string":"Faculty of Automation, Computer Science, Electrical and Electronic Engineering, \u201cDun\u0103rea de Jos\u201d University of Galati, 800008 Galati, Romania","institution_ids":["https://openalex.org/I3122748864"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Constantin Adrian Andrei","orcid":null},"institutions":[{"id":"https://openalex.org/I162051807","display_name":"Carol Davila University of Medicine and Pharmacy","ror":"https://ror.org/04fm87419","country_code":"RO","type":"education","lineage":["https://openalex.org/I162051807"]}],"countries":["RO"],"is_corresponding":false,"raw_author_name":"Constantin Adrian Andrei","raw_affiliation_strings":["Department of Orthopaedics, \u201cFoisor\u201d Clinical Hospital of Orthopaedics, Traumatology and Osteoarticular TB, 021382 Bucharest, Romania","Faculty of Medicine, The \u201cCarol Davila\u201d University of Medicine and Pharmacy, 050474 Bucharest, Romania"],"affiliations":[{"raw_affiliation_string":"Department of Orthopaedics, \u201cFoisor\u201d Clinical Hospital of Orthopaedics, Traumatology and Osteoarticular TB, 021382 Bucharest, Romania","institution_ids":[]},{"raw_affiliation_string":"Faculty of Medicine, The \u201cCarol Davila\u201d University of Medicine and Pharmacy, 050474 Bucharest, Romania","institution_ids":["https://openalex.org/I162051807"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Calina Maier","orcid":null},"institutions":[{"id":"https://openalex.org/I2799655038","display_name":"Clinical Emergency Hospital Bucharest","ror":"https://ror.org/03grprm46","country_code":"RO","type":"healthcare","lineage":["https://openalex.org/I2799655038"]},{"id":"https://openalex.org/I162051807","display_name":"Carol Davila University of Medicine and Pharmacy","ror":"https://ror.org/04fm87419","country_code":"RO","type":"education","lineage":["https://openalex.org/I162051807"]}],"countries":["RO"],"is_corresponding":false,"raw_author_name":"Calina Maier","raw_affiliation_strings":["Faculty of Medicine, The \u201cCarol Davila\u201d University of Medicine and Pharmacy, 050474 Bucharest, Romania","Panait Sirbu Obstetrics and Gynaecology Hospital Bucharest, 060251 Bucharest, Romania"],"affiliations":[{"raw_affiliation_string":"Faculty of Medicine, The \u201cCarol Davila\u201d University of Medicine and Pharmacy, 050474 Bucharest, Romania","institution_ids":["https://openalex.org/I162051807"]},{"raw_affiliation_string":"Panait Sirbu Obstetrics and Gynaecology Hospital Bucharest, 060251 Bucharest, Romania","institution_ids":["https://openalex.org/I2799655038"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Serban Dragosloveanu","orcid":"https://orcid.org/0000-0001-7403-1684"},"institutions":[{"id":"https://openalex.org/I162051807","display_name":"Carol Davila University of Medicine and Pharmacy","ror":"https://ror.org/04fm87419","country_code":"RO","type":"education","lineage":["https://openalex.org/I162051807"]}],"countries":["RO"],"is_corresponding":false,"raw_author_name":"Serban Dragosloveanu","raw_affiliation_strings":["Department of Orthopaedics, \u201cFoisor\u201d Clinical Hospital of Orthopaedics, Traumatology and Osteoarticular TB, 021382 Bucharest, Romania","Faculty of Medicine, The \u201cCarol Davila\u201d University of Medicine and Pharmacy, 050474 Bucharest, Romania"],"affiliations":[{"raw_affiliation_string":"Department of Orthopaedics, \u201cFoisor\u201d Clinical Hospital of Orthopaedics, Traumatology and Osteoarticular TB, 021382 Bucharest, Romania","institution_ids":[]},{"raw_affiliation_string":"Faculty of Medicine, The \u201cCarol Davila\u201d University of Medicine and Pharmacy, 050474 Bucharest, Romania","institution_ids":["https://openalex.org/I162051807"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Dana-Georgiana Nedelea","orcid":null},"institutions":[{"id":"https://openalex.org/I162051807","display_name":"Carol Davila University of Medicine and Pharmacy","ror":"https://ror.org/04fm87419","country_code":"RO","type":"education","lineage":["https://openalex.org/I162051807"]}],"countries":["RO"],"is_corresponding":false,"raw_author_name":"Dana-Georgiana Nedelea","raw_affiliation_strings":["Department of Orthopaedics, \u201cFoisor\u201d Clinical Hospital of Orthopaedics, Traumatology and Osteoarticular TB, 021382 Bucharest, Romania","Faculty of Medicine, The \u201cCarol Davila\u201d University of Medicine and Pharmacy, 050474 Bucharest, Romania"],"affiliations":[{"raw_affiliation_string":"Department of Orthopaedics, \u201cFoisor\u201d Clinical Hospital of Orthopaedics, Traumatology and Osteoarticular TB, 021382 Bucharest, Romania","institution_ids":[]},{"raw_affiliation_string":"Faculty of Medicine, The \u201cCarol Davila\u201d University of Medicine and Pharmacy, 050474 Bucharest, Romania","institution_ids":["https://openalex.org/I162051807"]}]},{"author_position":"last","author":{"id":null,"display_name":"Cristian Scheau","orcid":"https://orcid.org/0000-0001-7676-6393"},"institutions":[{"id":"https://openalex.org/I162051807","display_name":"Carol Davila University of Medicine and Pharmacy","ror":"https://ror.org/04fm87419","country_code":"RO","type":"education","lineage":["https://openalex.org/I162051807"]}],"countries":["RO"],"is_corresponding":false,"raw_author_name":"Cristian Scheau","raw_affiliation_strings":["Department of Radiology and Medical Imaging, \u201cFoisor\u201d Clinical Hospital of Orthopaedics, Traumatology and Osteoarticular TB, 021382 Bucharest, Romania","Faculty of Medicine, The \u201cCarol Davila\u201d University of Medicine and Pharmacy, 050474 Bucharest, Romania"],"affiliations":[{"raw_affiliation_string":"Department of Radiology and Medical Imaging, \u201cFoisor\u201d Clinical Hospital of Orthopaedics, Traumatology and Osteoarticular TB, 021382 Bucharest, Romania","institution_ids":[]},{"raw_affiliation_string":"Faculty of Medicine, The \u201cCarol Davila\u201d University of Medicine and Pharmacy, 050474 Bucharest, Romania","institution_ids":["https://openalex.org/I162051807"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":10,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I3122748864"],"apc_list":{"value":1600,"currency":"CHF","value_usd":1732},"apc_paid":{"value":1600,"currency":"CHF","value_usd":1732},"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.50233503,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"14","issue":"12","first_page":"530","last_page":"530"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.20630000531673431,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},"topics":[{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.20630000531673431,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T13910","display_name":"Computational and Text Analysis Methods","score":0.14000000059604645,"subfield":{"id":"https://openalex.org/subfields/3300","display_name":"General Social Sciences"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.11150000244379044,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/rubric","display_name":"Rubric","score":0.7904000282287598},{"id":"https://openalex.org/keywords/adjudication","display_name":"Adjudication","score":0.5358999967575073},{"id":"https://openalex.org/keywords/concordance","display_name":"Concordance","score":0.5198000073432922},{"id":"https://openalex.org/keywords/pairwise-comparison","display_name":"Pairwise comparison","score":0.49380001425743103},{"id":"https://openalex.org/keywords/robustness","display_name":"Robustness (evolution)","score":0.4805999994277954},{"id":"https://openalex.org/keywords/workflow","display_name":"Workflow","score":0.4375999867916107},{"id":"https://openalex.org/keywords/grading","display_name":"Grading (engineering)","score":0.4341999888420105},{"id":"https://openalex.org/keywords/closeness","display_name":"Closeness","score":0.4185999929904938}],"concepts":[{"id":"https://openalex.org/C111640148","wikidata":"https://www.wikidata.org/wiki/Q847349","display_name":"Rubric","level":2,"score":0.7904000282287598},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5424000024795532},{"id":"https://openalex.org/C204434341","wikidata":"https://www.wikidata.org/wiki/Q357789","display_name":"Adjudication","level":2,"score":0.5358999967575073},{"id":"https://openalex.org/C160798450","wikidata":"https://www.wikidata.org/wiki/Q4230870","display_name":"Concordance","level":2,"score":0.5198000073432922},{"id":"https://openalex.org/C184898388","wikidata":"https://www.wikidata.org/wiki/Q1435712","display_name":"Pairwise comparison","level":2,"score":0.49380001425743103},{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.4805999994277954},{"id":"https://openalex.org/C177212765","wikidata":"https://www.wikidata.org/wiki/Q627335","display_name":"Workflow","level":2,"score":0.4375999867916107},{"id":"https://openalex.org/C2777286243","wikidata":"https://www.wikidata.org/wiki/Q5591926","display_name":"Grading (engineering)","level":2,"score":0.4341999888420105},{"id":"https://openalex.org/C2779545769","wikidata":"https://www.wikidata.org/wiki/Q5135364","display_name":"Closeness","level":2,"score":0.4185999929904938},{"id":"https://openalex.org/C61863361","wikidata":"https://www.wikidata.org/wiki/Q470749","display_name":"Inter-rater reliability","level":3,"score":0.3959999978542328},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3675999939441681},{"id":"https://openalex.org/C27158222","wikidata":"https://www.wikidata.org/wiki/Q5532422","display_name":"Generalizability theory","level":2,"score":0.3474000096321106},{"id":"https://openalex.org/C2776434776","wikidata":"https://www.wikidata.org/wiki/Q19246213","display_name":"Domain adaptation","level":3,"score":0.3310000002384186},{"id":"https://openalex.org/C199521495","wikidata":"https://www.wikidata.org/wiki/Q181487","display_name":"Audit","level":2,"score":0.31700000166893005},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.28380000591278076},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.2800999879837036},{"id":"https://openalex.org/C183115368","wikidata":"https://www.wikidata.org/wiki/Q856577","display_name":"Weighting","level":2,"score":0.26660001277923584},{"id":"https://openalex.org/C117251300","wikidata":"https://www.wikidata.org/wiki/Q1849855","display_name":"Parametric statistics","level":2,"score":0.26440000534057617},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.250900000333786}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.3390/computers14120530","is_oa":true,"landing_page_url":"https://doi.org/10.3390/computers14120530","pdf_url":null,"source":{"id":"https://openalex.org/S4210228075","display_name":"Computers","issn_l":"2073-431X","issn":["2073-431X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310310987","host_organization_name":"Multidisciplinary Digital Publishing Institute","host_organization_lineage":["https://openalex.org/P4310310987"],"host_organization_lineage_names":["Multidisciplinary Digital Publishing Institute"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Computers","raw_type":"journal-article"},{"id":"pmh:oai:doaj.org/article:191532b576ee4c4cbb89744438aec639","is_oa":true,"landing_page_url":"https://doaj.org/article/191532b576ee4c4cbb89744438aec639","pdf_url":null,"source":{"id":"https://openalex.org/S112646816","display_name":"SHILAP Revista de lepidopterolog\u00eda","issn_l":"0300-5267","issn":["0300-5267","2340-4078"],"is_oa":true,"is_in_doaj":true,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Computers, Vol 14, Iss 12, p 530 (2025)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.3390/computers14120530","is_oa":true,"landing_page_url":"https://doi.org/10.3390/computers14120530","pdf_url":null,"source":{"id":"https://openalex.org/S4210228075","display_name":"Computers","issn_l":"2073-431X","issn":["2073-431X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310310987","host_organization_name":"Multidisciplinary Digital Publishing Institute","host_organization_lineage":["https://openalex.org/P4310310987"],"host_organization_lineage_names":["Multidisciplinary Digital Publishing Institute"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Computers","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":29,"referenced_works":["https://openalex.org/W2101105183","https://openalex.org/W2981731882","https://openalex.org/W3133543405","https://openalex.org/W3198690080","https://openalex.org/W4296467532","https://openalex.org/W4306407754","https://openalex.org/W4321786089","https://openalex.org/W4381599258","https://openalex.org/W4385571412","https://openalex.org/W4389408896","https://openalex.org/W4400771997","https://openalex.org/W4402671506","https://openalex.org/W4402909345","https://openalex.org/W4404781628","https://openalex.org/W4404781852","https://openalex.org/W4405153184","https://openalex.org/W4405239924","https://openalex.org/W4405916612","https://openalex.org/W4406327959","https://openalex.org/W4410550984","https://openalex.org/W4410768139","https://openalex.org/W4411119926","https://openalex.org/W4412370786","https://openalex.org/W4412995610","https://openalex.org/W4413025843","https://openalex.org/W4413072353","https://openalex.org/W4415210618","https://openalex.org/W4416035226","https://openalex.org/W4416037577"],"related_works":[],"abstract_inverted_index":{"Large":[0],"Language":[1],"Models":[2],"(LLMs)":[3],"are":[4,39,147,204],"increasingly":[5],"used":[6],"for":[7,26,153,290],"rubric-based":[8],"assessment,":[9],"yet":[10],"reliability":[11],"is":[12,129],"limited":[13],"by":[14,83,131,191],"instability,":[15],"bias,":[16],"and":[17,32,54,57,67,78,101,122,124,135,144,167,180,209,230,254,271,287,294,297],"weak":[18],"diagnostics.":[19],"We":[20,106],"present":[21],"EvalCouncil,":[22],"a":[23,33,60,98,150,196,215,224,242,250,258,267,284],"committee-and-chief":[24],"framework":[25],"rubric-guided":[27,244],"grading":[28,245,293],"with":[29,176,186,236,247],"auditable":[30],"traces":[31,146],"human":[34,251],"adjudication":[35,252],"baseline.":[36],"Our":[37],"objectives":[38],"to":[40,51,206,227],"(i)":[41],"characterize":[42],"domain":[43,160],"structure":[44],"in":[45,65,149,257],"Human\u2013LLM":[46,126,156],"alignment,":[47],"(ii)":[48],"assess":[49],"robustness":[50],"concordance":[52,118],"tolerance":[53,99,134,207,269],"panel":[55,140],"composition,":[56],"(iii)":[58],"derive":[59],"domain-adaptive":[61],"audit":[62],"policy":[63,268],"grounded":[64],"dispersion":[66,109,175,265],"chief\u2013panel":[68,117,272],"differences.":[69],"Authentic":[70],"student":[71],"responses":[72],"from":[73],"two":[74],"domains\u2013Computer":[75],"Networks":[76],"(CNs)":[77],"Machine":[79],"Learning":[80],"(ML)\u2013are":[81],"graded":[82],"multiple":[84],"heterogeneous":[85],"LLM":[86],"evaluators":[87],"using":[88],"identical":[89],"rubric":[90,238],"prompts.":[91],"A":[92],"designated":[93],"chief":[94],"arbitrator":[95],"operates":[96],"within":[97],"band":[100],"issues":[102],"the":[103,133,274],"final":[104],"grade.":[105],"quantify":[107],"within-panel":[108],"via":[110],"MPAD":[111],"(mean":[112],"pairwise":[113],"absolute":[114,120],"deviation),":[115],"measure":[116],"(e.g.,":[119],"error":[121],"bias),":[123],"compute":[125],"deviation.":[127],"Robustness":[128],"examined":[130],"sweeping":[132],"performing":[136],"leave-one-out":[137],"perturbations":[138],"of":[139,200],"composition.":[141],"All":[142],"outputs":[143],"reasoning":[145],"stored":[148],"graph":[151],"database":[152],"full":[154],"provenance.":[155],"alignment":[157],"exhibits":[158],"systematic":[159,298],"dependence:":[161],"ML":[162],"shows":[163,276],"tighter":[164],"central":[165],"tendency":[166],"shorter":[168],"upper":[169,178],"tails,":[170],"whereas":[171],"CN":[172],"displays":[173],"broader":[174],"heavier":[177],"tails":[179],"larger":[181],"extreme":[182],"spreads.":[183],"Disagreement":[184],"increases":[185],"item":[187],"difficulty":[188],"as":[189],"captured":[190],"MPAD,":[192],"concentrating":[193],"misalignment":[194],"on":[195],"relatively":[197],"small":[198],"subset":[199],"items.":[201],"These":[202],"patterns":[203],"stable":[205],"variation":[208],"single-grader":[210],"removals.":[211],"The":[212],"signals":[213],"support":[214],"practical":[216],"triage":[217],"policy:":[218],"accept":[219],"low-dispersion,":[220],"small-gap":[221],"items;":[222],"apply":[223],"brief":[225],"check":[226],"borderline":[228],"cases;":[229],"adjudicate":[231],"high-dispersion":[232],"or":[233],"large-gap":[234],"items":[235],"targeted":[237],"clarification.":[239],"EvalCouncil":[240],"instantiates":[241],"committee-and-chief,":[243],"workflow":[246],"committee":[248],"arbitration,":[249],"baseline,":[253],"graph-based":[255],"auditability":[256],"real":[259],"classroom":[260],"deployment.":[261],"By":[262],"linking":[263],"domain-aware":[264],"(MPAD),":[266],"dial,":[270],"discrepancy,":[273],"study":[275],"how":[277],"these":[278],"elements":[279],"can":[280],"be":[281],"combined":[282],"into":[283],"replicable,":[285],"auditable,":[286],"capacity-aware":[288],"approach":[289],"organizing":[291],"LLM-assisted":[292],"identifying":[295],"instability":[296],"misalignment,":[299],"while":[300],"maintaining":[301],"pedagogical":[302],"interpretability.":[303]},"counts_by_year":[],"updated_date":"2026-02-27T16:54:17.756197","created_date":"2025-12-04T00:00:00"}