{"id":"https://openalex.org/W4412995610","doi":"https://doi.org/10.3390/info16080652","title":"Diagnosing Bias and Instability in LLM Evaluation: A Scalable Pairwise Meta-Evaluator","display_name":"Diagnosing Bias and Instability in LLM Evaluation: A Scalable Pairwise Meta-Evaluator","publication_year":2025,"publication_date":"2025-07-31","ids":{"openalex":"https://openalex.org/W4412995610","doi":"https://doi.org/10.3390/info16080652"},"language":"en","primary_location":{"id":"doi:10.3390/info16080652","is_oa":true,"landing_page_url":"https://doi.org/10.3390/info16080652","pdf_url":"https://www.mdpi.com/2078-2489/16/8/652/pdf?version=1753955576","source":{"id":"https://openalex.org/S4210219776","display_name":"Information","issn_l":"2078-2489","issn":["2078-2489"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310310987","host_organization_name":"Multidisciplinary Digital Publishing Institute","host_organization_lineage":["https://openalex.org/P4310310987"],"host_organization_lineage_names":["Multidisciplinary Digital Publishing Institute"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Information","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://www.mdpi.com/2078-2489/16/8/652/pdf?version=1753955576","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5049418389","display_name":"C\u0103t\u0103lin Anghel","orcid":"https://orcid.org/0000-0002-1849-3072"},"institutions":[{"id":"https://openalex.org/I3122748864","display_name":"\"Dunarea de Jos\" University of Galati","ror":"https://ror.org/052sta926","country_code":"RO","type":"education","lineage":["https://openalex.org/I3122748864"]}],"countries":["RO"],"is_corresponding":true,"raw_author_name":"Catalin Anghel","raw_affiliation_strings":["Department of Computer Science and Information Technology, \u201cDun\u0103rea de Jos\u201d University of Galati, \u0218tiin\u021bei St. 2, 800146 Galati, Romania"],"raw_orcid":"https://orcid.org/0000-0002-1849-3072","affiliations":[{"raw_affiliation_string":"Department of Computer Science and Information Technology, \u201cDun\u0103rea de Jos\u201d University of Galati, \u0218tiin\u021bei St. 2, 800146 Galati, Romania","institution_ids":["https://openalex.org/I3122748864"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5119223040","display_name":"Andreea Alexandra Anghel","orcid":"https://orcid.org/0009-0008-2537-6713"},"institutions":[{"id":"https://openalex.org/I3122748864","display_name":"\"Dunarea de Jos\" University of Galati","ror":"https://ror.org/052sta926","country_code":"RO","type":"education","lineage":["https://openalex.org/I3122748864"]}],"countries":["RO"],"is_corresponding":false,"raw_author_name":"Andreea Alexandra Anghel","raw_affiliation_strings":["Faculty of Automation, Computer Science, Electrical and Electronic Engineering, \u201cDun\u0103rea de Jos\u201d University of Galati, 800008 Galati, Romania"],"raw_orcid":"https://orcid.org/0009-0008-2537-6713","affiliations":[{"raw_affiliation_string":"Faculty of Automation, Computer Science, Electrical and Electronic Engineering, \u201cDun\u0103rea de Jos\u201d University of Galati, 800008 Galati, Romania","institution_ids":["https://openalex.org/I3122748864"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5052924860","display_name":"Emilia Pecheanu","orcid":"https://orcid.org/0000-0003-1155-5274"},"institutions":[{"id":"https://openalex.org/I3122748864","display_name":"\"Dunarea de Jos\" University of Galati","ror":"https://ror.org/052sta926","country_code":"RO","type":"education","lineage":["https://openalex.org/I3122748864"]}],"countries":["RO"],"is_corresponding":true,"raw_author_name":"Emilia Pecheanu","raw_affiliation_strings":["Department of Computer Science and Information Technology, \u201cDun\u0103rea de Jos\u201d University of Galati, \u0218tiin\u021bei St. 2, 800146 Galati, Romania"],"raw_orcid":"https://orcid.org/0000-0003-1155-5274","affiliations":[{"raw_affiliation_string":"Department of Computer Science and Information Technology, \u201cDun\u0103rea de Jos\u201d University of Galati, \u0218tiin\u021bei St. 2, 800146 Galati, Romania","institution_ids":["https://openalex.org/I3122748864"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103039137","display_name":"Adina Cocu","orcid":"https://orcid.org/0000-0003-0935-4713"},"institutions":[{"id":"https://openalex.org/I3122748864","display_name":"\"Dunarea de Jos\" University of Galati","ror":"https://ror.org/052sta926","country_code":"RO","type":"education","lineage":["https://openalex.org/I3122748864"]}],"countries":["RO"],"is_corresponding":false,"raw_author_name":"Adina Cocu","raw_affiliation_strings":["Department of Computer Science and Information Technology, \u201cDun\u0103rea de Jos\u201d University of Galati, \u0218tiin\u021bei St. 2, 800146 Galati, Romania"],"raw_orcid":"https://orcid.org/0000-0003-0935-4713","affiliations":[{"raw_affiliation_string":"Department of Computer Science and Information Technology, \u201cDun\u0103rea de Jos\u201d University of Galati, \u0218tiin\u021bei St. 2, 800146 Galati, Romania","institution_ids":["https://openalex.org/I3122748864"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5077496241","display_name":"Adrian Istrate","orcid":"https://orcid.org/0000-0002-1317-6676"},"institutions":[{"id":"https://openalex.org/I3122748864","display_name":"\"Dunarea de Jos\" University of Galati","ror":"https://ror.org/052sta926","country_code":"RO","type":"education","lineage":["https://openalex.org/I3122748864"]}],"countries":["RO"],"is_corresponding":false,"raw_author_name":"Adrian Istrate","raw_affiliation_strings":["Department of Computer Science and Information Technology, \u201cDun\u0103rea de Jos\u201d University of Galati, \u0218tiin\u021bei St. 2, 800146 Galati, Romania"],"raw_orcid":"https://orcid.org/0000-0002-1317-6676","affiliations":[{"raw_affiliation_string":"Department of Computer Science and Information Technology, \u201cDun\u0103rea de Jos\u201d University of Galati, \u0218tiin\u021bei St. 2, 800146 Galati, Romania","institution_ids":["https://openalex.org/I3122748864"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5110362291","display_name":"Constantin Adrian Andrei","orcid":"https://orcid.org/0000-0001-5595-1135"},"institutions":[{"id":"https://openalex.org/I2799655038","display_name":"Clinical Emergency Hospital Bucharest","ror":"https://ror.org/03grprm46","country_code":"RO","type":"healthcare","lineage":["https://openalex.org/I2799655038"]}],"countries":["RO"],"is_corresponding":false,"raw_author_name":"Constantin Adrian Andrei","raw_affiliation_strings":["\u201cFoisor\u201d Clinical Hospital of Orthopaedics, Traumatology and Osteoarticular TB, 021382 Bucharest, Romania","\"Foisor\" Clinical Hospital of Orthopaedics, Traumatology and Osteoarticular TB, 021382 Bucharest, Romania"],"raw_orcid":"https://orcid.org/0000-0001-5595-1135","affiliations":[{"raw_affiliation_string":"\u201cFoisor\u201d Clinical Hospital of Orthopaedics, Traumatology and Osteoarticular TB, 021382 Bucharest, Romania","institution_ids":["https://openalex.org/I2799655038"]},{"raw_affiliation_string":"\"Foisor\" Clinical Hospital of Orthopaedics, Traumatology and Osteoarticular TB, 021382 Bucharest, Romania","institution_ids":["https://openalex.org/I2799655038"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5049418389","https://openalex.org/A5052924860"],"corresponding_institution_ids":["https://openalex.org/I3122748864"],"apc_list":{"value":1400,"currency":"CHF","value_usd":1515},"apc_paid":{"value":1400,"currency":"CHF","value_usd":1515},"fwci":16.5728,"has_fulltext":true,"cited_by_count":10,"citation_normalized_percentile":{"value":0.98967758,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":99},"biblio":{"volume":"16","issue":"8","first_page":"652","last_page":"652"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9922000169754028,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9922000169754028,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13629","display_name":"Text Readability and Simplification","score":0.9919000267982483,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9667999744415283,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/pairwise-comparison","display_name":"Pairwise comparison","score":0.7679948210716248},{"id":"https://openalex.org/keywords/instability","display_name":"Instability","score":0.5213680863380432},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.4793154001235962},{"id":"https://openalex.org/keywords/scalability","display_name":"Scalability","score":0.4789212942123413},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.23287206888198853},{"id":"https://openalex.org/keywords/physics","display_name":"Physics","score":0.09572619199752808},{"id":"https://openalex.org/keywords/database","display_name":"Database","score":0.07985261082649231}],"concepts":[{"id":"https://openalex.org/C184898388","wikidata":"https://www.wikidata.org/wiki/Q1435712","display_name":"Pairwise comparison","level":2,"score":0.7679948210716248},{"id":"https://openalex.org/C207821765","wikidata":"https://www.wikidata.org/wiki/Q405372","display_name":"Instability","level":2,"score":0.5213680863380432},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4793154001235962},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.4789212942123413},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.23287206888198853},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.09572619199752808},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.07985261082649231},{"id":"https://openalex.org/C57879066","wikidata":"https://www.wikidata.org/wiki/Q41217","display_name":"Mechanics","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.3390/info16080652","is_oa":true,"landing_page_url":"https://doi.org/10.3390/info16080652","pdf_url":"https://www.mdpi.com/2078-2489/16/8/652/pdf?version=1753955576","source":{"id":"https://openalex.org/S4210219776","display_name":"Information","issn_l":"2078-2489","issn":["2078-2489"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310310987","host_organization_name":"Multidisciplinary Digital Publishing Institute","host_organization_lineage":["https://openalex.org/P4310310987"],"host_organization_lineage_names":["Multidisciplinary Digital Publishing Institute"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Information","raw_type":"journal-article"},{"id":"pmh:oai:doaj.org/article:8dd12c29657742e99560d9eb15b53ab0","is_oa":true,"landing_page_url":"https://doaj.org/article/8dd12c29657742e99560d9eb15b53ab0","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Information, Vol 16, Iss 8, p 652 (2025)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.3390/info16080652","is_oa":true,"landing_page_url":"https://doi.org/10.3390/info16080652","pdf_url":"https://www.mdpi.com/2078-2489/16/8/652/pdf?version=1753955576","source":{"id":"https://openalex.org/S4210219776","display_name":"Information","issn_l":"2078-2489","issn":["2078-2489"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310310987","host_organization_name":"Multidisciplinary Digital Publishing Institute","host_organization_lineage":["https://openalex.org/P4310310987"],"host_organization_lineage_names":["Multidisciplinary Digital Publishing Institute"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Information","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4412995610.pdf","grobid_xml":"https://content.openalex.org/works/W4412995610.grobid-xml"},"referenced_works_count":16,"referenced_works":["https://openalex.org/W1985514943","https://openalex.org/W4353112996","https://openalex.org/W4385849309","https://openalex.org/W4395064689","https://openalex.org/W4399750638","https://openalex.org/W4401024643","https://openalex.org/W4402670690","https://openalex.org/W4402671506","https://openalex.org/W4402684217","https://openalex.org/W4403764183","https://openalex.org/W4404783497","https://openalex.org/W4405183979","https://openalex.org/W4408598315","https://openalex.org/W4410968179","https://openalex.org/W4411113160","https://openalex.org/W6855757595"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2487162673","https://openalex.org/W2793211469","https://openalex.org/W2949152769","https://openalex.org/W4372354731","https://openalex.org/W2942366970","https://openalex.org/W2807634898","https://openalex.org/W1692008701"],"abstract_inverted_index":{"The":[0,61,103,171,207],"evaluation":[1,49,156,202],"of":[2,82,124],"large":[3],"language":[4,59,223],"models":[5,98,182],"(LLMs)":[6],"increasingly":[7],"relies":[8],"on":[9],"other":[10],"LLMs":[11],"acting":[12],"as":[13],"automated":[14],"judges.":[15],"While":[16],"this":[17,229],"approach":[18],"offers":[19,210],"scalability":[20],"and":[21,32,45,72,85,168,189,218,238],"efficiency,":[22],"it":[23],"raises":[24],"serious":[25],"concerns":[26],"regarding":[27],"evaluator":[28,83,153,216],"reliability,":[29],"positional":[30,43,133],"bias,":[31],"ranking":[33,219],"stability.":[34],"This":[35],"paper":[36],"presents":[37],"a":[38,108,161,177,185,211],"scalable":[39],"framework":[40,173,209],"for":[41,180,187,199,214],"diagnosing":[42,215],"bias":[44,217],"instability":[46,220],"in":[47,160,221,245],"LLM-based":[48,192],"by":[50,56],"using":[51,99,235],"controlled":[52],"pairwise":[53,90],"comparisons":[54,65,91],"judged":[55],"multiple":[57],"independent":[58],"models.":[60],"system":[62],"supports":[63],"mirrored":[64,128],"with":[66,117],"reversed":[67,126],"response":[68,129],"order,":[69,130],"prompt":[70],"injection,":[71],"surface-level":[73],"perturbations":[74],"(e.g.,":[75],"paraphrasing,":[76],"lexical":[77],"noise),":[78],"enabling":[79,165],"fine-grained":[80],"analysis":[81,137],"consistency":[84,119],"verdict":[86],"robustness.":[87],"Over":[88],"3600":[89],"were":[92,158],"conducted":[93],"across":[94,120,146],"five":[95],"instruction-tuned":[96],"open-weight":[97],"ten":[100],"open-ended":[101,222],"prompts.":[102],"top-performing":[104],"model":[105,142],"(gemma:7b-instruct)":[106],"achieved":[107],"66.5%":[109],"win":[110],"rate.":[111],"Evaluator":[112],"agreement":[113],"was":[114],"uniformly":[115],"high,":[116],"100%":[118],"judges,":[121],"yet":[122],"48.4%":[123],"verdicts":[125],"under":[127],"indicating":[131],"strong":[132],"bias.":[134],"Kendall\u2019s":[135],"Tau":[136],"further":[138],"showed":[139],"that":[140,149],"local":[141],"rankings":[143],"varied":[144],"substantially":[145],"prompts,":[147],"suggesting":[148],"semantic":[150],"context":[151],"influences":[152],"judgment.":[154],"All":[155],"traces":[157],"stored":[159],"graph":[162],"database":[163],"(Neo4j),":[164],"structured":[166],"querying":[167],"longitudinal":[169],"analysis.":[170],"proposed":[172,208],"provides":[174],"not":[175],"only":[176],"diagnostic":[178],"lens":[179],"benchmarking":[181,205],"but":[183],"also":[184],"blueprint":[186],"fairer":[188],"more":[190],"interpretable":[191],"evaluation.":[193],"These":[194],"findings":[195],"underscore":[196],"the":[197],"need":[198],"structure-aware,":[200],"perturbation-resilient":[201],"pipelines":[203],"when":[204],"LLMs.":[206],"reproducible":[212],"path":[213],"tasks.":[224],"Future":[225],"work":[226],"will":[227],"apply":[228],"methodology":[230],"to":[231,241],"educational":[232],"assessment":[233],"tasks,":[234],"rubric-based":[236],"scoring":[237],"graph-based":[239],"traceability":[240],"evaluate":[242],"student":[243],"responses":[244],"technical":[246],"domains.":[247]},"counts_by_year":[{"year":2026,"cited_by_count":5},{"year":2025,"cited_by_count":5}],"updated_date":"2026-06-25T08:15:23.626066","created_date":"2025-10-10T00:00:00"}