{"id":"https://openalex.org/W4416093678","doi":"https://doi.org/10.3390/informatics12040123","title":"GraderAssist: A Graph-Based Multi-LLM Framework for Transparent and Reproducible Automated Evaluation","display_name":"GraderAssist: A Graph-Based Multi-LLM Framework for Transparent and Reproducible Automated Evaluation","publication_year":2025,"publication_date":"2025-11-09","ids":{"openalex":"https://openalex.org/W4416093678","doi":"https://doi.org/10.3390/informatics12040123"},"language":"en","primary_location":{"id":"doi:10.3390/informatics12040123","is_oa":true,"landing_page_url":"https://doi.org/10.3390/informatics12040123","pdf_url":"https://www.mdpi.com/2227-9709/12/4/123/pdf?version=1762675291","source":{"id":"https://openalex.org/S2738238905","display_name":"Informatics","issn_l":"2227-9709","issn":["2227-9709"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310310987","host_organization_name":"Multidisciplinary Digital Publishing Institute","host_organization_lineage":["https://openalex.org/P4310310987"],"host_organization_lineage_names":["Multidisciplinary Digital Publishing Institute"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Informatics","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://www.mdpi.com/2227-9709/12/4/123/pdf?version=1762675291","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5049418389","display_name":"C\u0103t\u0103lin Anghel","orcid":"https://orcid.org/0000-0002-1849-3072"},"institutions":[{"id":"https://openalex.org/I3122748864","display_name":"\"Dunarea de Jos\" University of Galati","ror":"https://ror.org/052sta926","country_code":"RO","type":"education","lineage":["https://openalex.org/I3122748864"]}],"countries":["RO"],"is_corresponding":true,"raw_author_name":"Catalin Anghel","raw_affiliation_strings":["Department of Computer Science and Information Technology, \u201cDun\u0103rea de Jos\u201d University of Galati, \u0218tiin\u021bei St. 2, 800201 Galati, Romania"],"affiliations":[{"raw_affiliation_string":"Department of Computer Science and Information Technology, \u201cDun\u0103rea de Jos\u201d University of Galati, \u0218tiin\u021bei St. 2, 800201 Galati, Romania","institution_ids":["https://openalex.org/I3122748864"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5119223040","display_name":"Andreea Alexandra Anghel","orcid":"https://orcid.org/0009-0008-2537-6713"},"institutions":[{"id":"https://openalex.org/I3122748864","display_name":"\"Dunarea de Jos\" University of Galati","ror":"https://ror.org/052sta926","country_code":"RO","type":"education","lineage":["https://openalex.org/I3122748864"]}],"countries":["RO"],"is_corresponding":false,"raw_author_name":"Andreea Alexandra Anghel","raw_affiliation_strings":["Computer Science and Information Technology Program, Faculty of Automation, Computer Science, Electrical and Electronic Engineering, \u201cDun\u0103rea de Jos\u201d University of Galati, 800201 Galati, Romania"],"affiliations":[{"raw_affiliation_string":"Computer Science and Information Technology Program, Faculty of Automation, Computer Science, Electrical and Electronic Engineering, \u201cDun\u0103rea de Jos\u201d University of Galati, 800201 Galati, Romania","institution_ids":["https://openalex.org/I3122748864"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5052924860","display_name":"Emilia Pecheanu","orcid":"https://orcid.org/0000-0003-1155-5274"},"institutions":[{"id":"https://openalex.org/I3122748864","display_name":"\"Dunarea de Jos\" University of Galati","ror":"https://ror.org/052sta926","country_code":"RO","type":"education","lineage":["https://openalex.org/I3122748864"]}],"countries":["RO"],"is_corresponding":false,"raw_author_name":"Emilia Pecheanu","raw_affiliation_strings":["Department of Computer Science and Information Technology, \u201cDun\u0103rea de Jos\u201d University of Galati, \u0218tiin\u021bei St. 2, 800201 Galati, Romania"],"affiliations":[{"raw_affiliation_string":"Department of Computer Science and Information Technology, \u201cDun\u0103rea de Jos\u201d University of Galati, \u0218tiin\u021bei St. 2, 800201 Galati, Romania","institution_ids":["https://openalex.org/I3122748864"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103039137","display_name":"Adina Cocu","orcid":"https://orcid.org/0000-0003-0935-4713"},"institutions":[{"id":"https://openalex.org/I3122748864","display_name":"\"Dunarea de Jos\" University of Galati","ror":"https://ror.org/052sta926","country_code":"RO","type":"education","lineage":["https://openalex.org/I3122748864"]}],"countries":["RO"],"is_corresponding":false,"raw_author_name":"Adina Cocu","raw_affiliation_strings":["Department of Computer Science and Information Technology, \u201cDun\u0103rea de Jos\u201d University of Galati, \u0218tiin\u021bei St. 2, 800201 Galati, Romania"],"affiliations":[{"raw_affiliation_string":"Department of Computer Science and Information Technology, \u201cDun\u0103rea de Jos\u201d University of Galati, \u0218tiin\u021bei St. 2, 800201 Galati, Romania","institution_ids":["https://openalex.org/I3122748864"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5110800740","display_name":"Marian Cr\u0103ciun","orcid":"https://orcid.org/0009-0006-9970-2556"},"institutions":[{"id":"https://openalex.org/I3122748864","display_name":"\"Dunarea de Jos\" University of Galati","ror":"https://ror.org/052sta926","country_code":"RO","type":"education","lineage":["https://openalex.org/I3122748864"]}],"countries":["RO"],"is_corresponding":true,"raw_author_name":"Marian Viorel Craciun","raw_affiliation_strings":["Department of Computer Science and Information Technology, \u201cDun\u0103rea de Jos\u201d University of Galati, \u0218tiin\u021bei St. 2, 800201 Galati, Romania"],"affiliations":[{"raw_affiliation_string":"Department of Computer Science and Information Technology, \u201cDun\u0103rea de Jos\u201d University of Galati, \u0218tiin\u021bei St. 2, 800201 Galati, Romania","institution_ids":["https://openalex.org/I3122748864"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5115061405","display_name":"Paul Iacobescu","orcid":"https://orcid.org/0009-0009-7539-4159"},"institutions":[{"id":"https://openalex.org/I3122748864","display_name":"\"Dunarea de Jos\" University of Galati","ror":"https://ror.org/052sta926","country_code":"RO","type":"education","lineage":["https://openalex.org/I3122748864"]}],"countries":["RO"],"is_corresponding":false,"raw_author_name":"Paul Iacobescu","raw_affiliation_strings":["Doctoral School, \u201cDun\u0103rea de Jos\u201d University of Galati, 800201 Galati, Romania"],"affiliations":[{"raw_affiliation_string":"Doctoral School, \u201cDun\u0103rea de Jos\u201d University of Galati, 800201 Galati, Romania","institution_ids":["https://openalex.org/I3122748864"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5120336926","display_name":"Antonio Stefan Balau","orcid":"https://orcid.org/0009-0002-5261-2060"},"institutions":[{"id":"https://openalex.org/I3122748864","display_name":"\"Dunarea de Jos\" University of Galati","ror":"https://ror.org/052sta926","country_code":"RO","type":"education","lineage":["https://openalex.org/I3122748864"]}],"countries":["RO"],"is_corresponding":false,"raw_author_name":"Antonio Stefan Balau","raw_affiliation_strings":["Computer Science and Information Technology Program, Faculty of Automation, Computer Science, Electrical and Electronic Engineering, \u201cDun\u0103rea de Jos\u201d University of Galati, 800201 Galati, Romania"],"affiliations":[{"raw_affiliation_string":"Computer Science and Information Technology Program, Faculty of Automation, Computer Science, Electrical and Electronic Engineering, \u201cDun\u0103rea de Jos\u201d University of Galati, 800201 Galati, Romania","institution_ids":["https://openalex.org/I3122748864"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5110362291","display_name":"Constantin Adrian Andrei","orcid":null},"institutions":[{"id":"https://openalex.org/I2799655038","display_name":"Clinical Emergency Hospital Bucharest","ror":"https://ror.org/03grprm46","country_code":"RO","type":"healthcare","lineage":["https://openalex.org/I2799655038"]}],"countries":["RO"],"is_corresponding":false,"raw_author_name":"Constantin Adrian Andrei","raw_affiliation_strings":["\u201cFoisor\u201d Clinical Hospital of Orthopaedics, Traumatology and Osteoarticular TB, 021382 Bucharest, Romania","\"Foisor\" Clinical Hospital of Orthopaedics, Traumatology and Osteoarticular TB, 021382 Bucharest, Romania"],"affiliations":[{"raw_affiliation_string":"\u201cFoisor\u201d Clinical Hospital of Orthopaedics, Traumatology and Osteoarticular TB, 021382 Bucharest, Romania","institution_ids":["https://openalex.org/I2799655038"]},{"raw_affiliation_string":"\"Foisor\" Clinical Hospital of Orthopaedics, Traumatology and Osteoarticular TB, 021382 Bucharest, Romania","institution_ids":["https://openalex.org/I2799655038"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":8,"corresponding_author_ids":["https://openalex.org/A5049418389","https://openalex.org/A5110800740"],"corresponding_institution_ids":["https://openalex.org/I3122748864"],"apc_list":{"value":1600,"currency":"CHF","value_usd":1732},"apc_paid":{"value":1600,"currency":"CHF","value_usd":1732},"fwci":5.399,"has_fulltext":true,"cited_by_count":2,"citation_normalized_percentile":{"value":0.95958428,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":99},"biblio":{"volume":"12","issue":"4","first_page":"123","last_page":"123"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11902","display_name":"Intelligent Tutoring Systems and Adaptive Learning","score":0.3562000095844269,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11902","display_name":"Intelligent Tutoring Systems and Adaptive Learning","score":0.3562000095844269,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11122","display_name":"Online Learning and Analytics","score":0.06520000100135803,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.04899999871850014,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/rubric","display_name":"Rubric","score":0.8774999976158142},{"id":"https://openalex.org/keywords/interpretability","display_name":"Interpretability","score":0.8769999742507935},{"id":"https://openalex.org/keywords/argumentative","display_name":"Argumentative","score":0.5583999752998352},{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.4936000108718872},{"id":"https://openalex.org/keywords/variable","display_name":"Variable (mathematics)","score":0.3734000027179718},{"id":"https://openalex.org/keywords/graph","display_name":"Graph","score":0.31220000982284546},{"id":"https://openalex.org/keywords/construct","display_name":"Construct (python library)","score":0.31139999628067017}],"concepts":[{"id":"https://openalex.org/C111640148","wikidata":"https://www.wikidata.org/wiki/Q847349","display_name":"Rubric","level":2,"score":0.8774999976158142},{"id":"https://openalex.org/C2781067378","wikidata":"https://www.wikidata.org/wiki/Q17027399","display_name":"Interpretability","level":2,"score":0.8769999742507935},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.725600004196167},{"id":"https://openalex.org/C2781306805","wikidata":"https://www.wikidata.org/wiki/Q4789761","display_name":"Argumentative","level":2,"score":0.5583999752998352},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.4936000108718872},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.38929998874664307},{"id":"https://openalex.org/C182365436","wikidata":"https://www.wikidata.org/wiki/Q50701","display_name":"Variable (mathematics)","level":2,"score":0.3734000027179718},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.3725000023841858},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3682999908924103},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.35179999470710754},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.34060001373291016},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.33239999413490295},{"id":"https://openalex.org/C132525143","wikidata":"https://www.wikidata.org/wiki/Q141488","display_name":"Graph","level":2,"score":0.31220000982284546},{"id":"https://openalex.org/C2780801425","wikidata":"https://www.wikidata.org/wiki/Q5164392","display_name":"Construct (python library)","level":2,"score":0.31139999628067017},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.2953999936580658},{"id":"https://openalex.org/C36464697","wikidata":"https://www.wikidata.org/wiki/Q451553","display_name":"Visualization","level":2,"score":0.29409998655319214},{"id":"https://openalex.org/C2777898490","wikidata":"https://www.wikidata.org/wiki/Q17157236","display_name":"Writing assessment","level":2,"score":0.28290000557899475},{"id":"https://openalex.org/C105002631","wikidata":"https://www.wikidata.org/wiki/Q4833645","display_name":"Subject-matter expert","level":3,"score":0.2786000072956085},{"id":"https://openalex.org/C42525527","wikidata":"https://www.wikidata.org/wiki/Q1209955","display_name":"Formative assessment","level":2,"score":0.27379998564720154},{"id":"https://openalex.org/C172367668","wikidata":"https://www.wikidata.org/wiki/Q6504956","display_name":"Data visualization","level":3,"score":0.25850000977516174},{"id":"https://openalex.org/C115901376","wikidata":"https://www.wikidata.org/wiki/Q184199","display_name":"Automation","level":2,"score":0.25200000405311584}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.3390/informatics12040123","is_oa":true,"landing_page_url":"https://doi.org/10.3390/informatics12040123","pdf_url":"https://www.mdpi.com/2227-9709/12/4/123/pdf?version=1762675291","source":{"id":"https://openalex.org/S2738238905","display_name":"Informatics","issn_l":"2227-9709","issn":["2227-9709"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310310987","host_organization_name":"Multidisciplinary Digital Publishing Institute","host_organization_lineage":["https://openalex.org/P4310310987"],"host_organization_lineage_names":["Multidisciplinary Digital Publishing Institute"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Informatics","raw_type":"journal-article"},{"id":"pmh:oai:doaj.org/article:6743f4f72aa94cf7a01e17e049eed0a1","is_oa":true,"landing_page_url":"https://doaj.org/article/6743f4f72aa94cf7a01e17e049eed0a1","pdf_url":null,"source":{"id":"https://openalex.org/S112646816","display_name":"SHILAP Revista de lepidopterolog\u00eda","issn_l":"0300-5267","issn":["0300-5267","2340-4078"],"is_oa":true,"is_in_doaj":true,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Informatics, Vol 12, Iss 4, p 123 (2025)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.3390/informatics12040123","is_oa":true,"landing_page_url":"https://doi.org/10.3390/informatics12040123","pdf_url":"https://www.mdpi.com/2227-9709/12/4/123/pdf?version=1762675291","source":{"id":"https://openalex.org/S2738238905","display_name":"Informatics","issn_l":"2227-9709","issn":["2227-9709"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310310987","host_organization_name":"Multidisciplinary Digital Publishing Institute","host_organization_lineage":["https://openalex.org/P4310310987"],"host_organization_lineage_names":["Multidisciplinary Digital Publishing Institute"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Informatics","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4416093678.pdf","grobid_xml":"https://content.openalex.org/works/W4416093678.grobid-xml"},"referenced_works_count":30,"referenced_works":["https://openalex.org/W2031235730","https://openalex.org/W2078202630","https://openalex.org/W2078483536","https://openalex.org/W2970641574","https://openalex.org/W3139502814","https://openalex.org/W4281695557","https://openalex.org/W4311974353","https://openalex.org/W4381599258","https://openalex.org/W4389228739","https://openalex.org/W4400009090","https://openalex.org/W4400771997","https://openalex.org/W4402671506","https://openalex.org/W4403764183","https://openalex.org/W4404568502","https://openalex.org/W4404781852","https://openalex.org/W4405183979","https://openalex.org/W4405755255","https://openalex.org/W4405916612","https://openalex.org/W4406327959","https://openalex.org/W4406668803","https://openalex.org/W4408218174","https://openalex.org/W4410550984","https://openalex.org/W4410862854","https://openalex.org/W4411113160","https://openalex.org/W4411429801","https://openalex.org/W4411962008","https://openalex.org/W4412750616","https://openalex.org/W4412886643","https://openalex.org/W4413909651","https://openalex.org/W4413928543"],"related_works":[],"abstract_inverted_index":{"Background":[0],"and":[1,16,43,50,66,81,93,110,114,125,137,161,185,196,215],"objectives:":[2],"Automated":[3],"evaluation":[4,209],"of":[5,75,199,206],"open-ended":[6],"responses":[7,77],"remains":[8],"a":[9,57,73,117,189],"persistent":[10],"challenge,":[11],"particularly":[12],"when":[13],"consistency,":[14],"transparency,":[15,195],"reproducibility":[17],"are":[18],"required.":[19],"While":[20],"large":[21],"language":[22],"models":[23,141],"(LLMs)":[24],"have":[25],"shown":[26],"promise":[27],"in":[28,40,116,134,213],"rubric-based":[29],"evaluation,":[30,182],"their":[31],"reliability":[32],"across":[33],"multiple":[34],"evaluators":[35],"is":[36],"still":[37],"uncertain.":[38],"Variability":[39],"scoring,":[41],"feedback,":[42,109],"rubric":[44,163],"adherence":[45],"raises":[46],"concerns":[47],"about":[48],"interpretability":[49],"system":[51],"robustness.":[52],"This":[53],"study":[54],"introduces":[55],"GraderAssist,":[56],"graph-based,":[58],"rubric-guided,":[59],"multi-LLM":[60],"framework":[61],"designed":[62],"to":[63,78],"ensure":[64],"transparent":[65],"reproducible":[67],"automated":[68,208],"evaluation.":[69],"Methods:":[70],"GraderAssist":[71,179],"evaluates":[72],"dataset":[74],"220":[76],"both":[79],"technical":[80,170],"argumentative":[82,176],"questions,":[83],"collected":[84],"from":[85],"undergraduate":[86],"computer":[87],"science":[88],"courses.":[89],"Six":[90],"open-source":[91],"LLMs":[92],"GPT-4":[94],"(as":[95],"expert":[96],"reference)":[97],"independently":[98],"scored":[99],"each":[100],"response":[101],"using":[102,154],"two":[103],"predefined":[104],"rubrics.":[105],"All":[106],"outputs\u2014including":[107],"scores,":[108],"metadata\u2014were":[111],"parsed,":[112],"validated,":[113],"stored":[115],"Neo4j":[118],"graph":[119],"database,":[120],"enabling":[121],"structured":[122],"querying,":[123],"traceability,":[124],"longitudinal":[126],"analysis.":[127],"Results:":[128],"Cross-model":[129],"analysis":[130,153,198],"revealed":[131],"systematic":[132],"differences":[133],"scoring":[135],"behavior":[136],"feedback":[138,159],"generation.":[139],"Some":[140],"produced":[142],"more":[143],"generous":[144],"evaluations,":[145],"while":[146],"others":[147],"aligned":[148],"closely":[149],"with":[150,211],"GPT-4.":[151],"Semantic":[152],"Sentence-BERT":[155],"embeddings":[156],"highlighted":[157],"distinctive":[158],"styles":[160],"variable":[162],"adherence.":[164],"Inter-model":[165],"agreement":[166],"was":[167],"stronger":[168],"for":[169,175],"criteria":[171],"but":[172],"diverged":[173],"substantially":[174],"tasks.":[177],"Originality:":[178],"integrates":[180],"rubric-guided":[181],"multi-model":[183],"comparison,":[184],"graph-based":[186],"storage":[187],"into":[188],"unified":[190],"pipeline.":[191],"By":[192],"emphasizing":[193],"reproducibility,":[194],"fine-grained":[197],"evaluator":[200],"behavior,":[201],"it":[202],"advances":[203],"the":[204],"design":[205],"interpretable":[207],"systems":[210],"applications":[212],"education":[214],"beyond.":[216]},"counts_by_year":[{"year":2026,"cited_by_count":2}],"updated_date":"2026-03-12T06:13:28.667946","created_date":"2025-11-10T00:00:00"}