{"id":"https://openalex.org/W7130581838","doi":"https://doi.org/10.48550/arxiv.2602.16610","title":"Who can we trust? LLM-as-a-jury for Comparative Assessment","display_name":"Who can we trust? LLM-as-a-jury for Comparative Assessment","publication_year":2026,"publication_date":"2026-02-18","ids":{"openalex":"https://openalex.org/W7130581838","doi":"https://doi.org/10.48550/arxiv.2602.16610"},"language":null,"primary_location":{"id":"pmh:doi:10.48550/arxiv.2602.16610","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4406922384","display_name":"Open MIND","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"publisher-specific-oa","license_id":"https://openalex.org/licenses/publisher-specific-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":null,"any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5081154184","display_name":"Mengjie Qian","orcid":"https://orcid.org/0000-0001-9768-2594"},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Qian, Mengjie","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5126433083","display_name":"Guangzhi Sun","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sun, Guangzhi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5126400649","display_name":"Mark J. F. Gales","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gales, Mark J. F.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5111076409","display_name":"Kate Knill","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Knill, Kate M.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5081154184"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.2587999999523163,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.2587999999523163,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10467","display_name":"Psychometric Methodologies and Testing","score":0.11259999871253967,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.06319999694824219,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/discriminator","display_name":"Discriminator","score":0.8500999808311462},{"id":"https://openalex.org/keywords/pairwise-comparison","display_name":"Pairwise comparison","score":0.7943000197410583},{"id":"https://openalex.org/keywords/consistency","display_name":"Consistency (knowledge bases)","score":0.6798999905586243},{"id":"https://openalex.org/keywords/aggregate","display_name":"Aggregate (composite)","score":0.650600016117096},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.6381999850273132},{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.609499990940094},{"id":"https://openalex.org/keywords/calibration","display_name":"Calibration","score":0.5476999878883362}],"concepts":[{"id":"https://openalex.org/C2779803651","wikidata":"https://www.wikidata.org/wiki/Q5282088","display_name":"Discriminator","level":3,"score":0.8500999808311462},{"id":"https://openalex.org/C184898388","wikidata":"https://www.wikidata.org/wiki/Q1435712","display_name":"Pairwise comparison","level":2,"score":0.7943000197410583},{"id":"https://openalex.org/C2776436953","wikidata":"https://www.wikidata.org/wiki/Q5163215","display_name":"Consistency (knowledge bases)","level":2,"score":0.6798999905586243},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6700000166893005},{"id":"https://openalex.org/C4679612","wikidata":"https://www.wikidata.org/wiki/Q866298","display_name":"Aggregate (composite)","level":2,"score":0.650600016117096},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.6381999850273132},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.609499990940094},{"id":"https://openalex.org/C165838908","wikidata":"https://www.wikidata.org/wiki/Q736777","display_name":"Calibration","level":2,"score":0.5476999878883362},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4918999969959259},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.474700003862381},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3553999960422516},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.3366999924182892},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.31610000133514404},{"id":"https://openalex.org/C2778029271","wikidata":"https://www.wikidata.org/wiki/Q5421931","display_name":"Extension (predicate logic)","level":2,"score":0.31130000948905945},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.28519999980926514},{"id":"https://openalex.org/C65660741","wikidata":"https://www.wikidata.org/wiki/Q3952743","display_name":"Score","level":2,"score":0.2851000130176544},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.262800008058548},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.26100000739097595},{"id":"https://openalex.org/C168167062","wikidata":"https://www.wikidata.org/wiki/Q1117970","display_name":"Component (thermodynamics)","level":2,"score":0.2517000138759613}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:doi:10.48550/arxiv.2602.16610","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4406922384","display_name":"Open MIND","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"publisher-specific-oa","license_id":"https://openalex.org/licenses/publisher-specific-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"},{"id":"doi:10.48550/arxiv.2602.16610","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2602.16610","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"pmh:doi:10.48550/arxiv.2602.16610","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4406922384","display_name":"Open MIND","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"publisher-specific-oa","license_id":"https://openalex.org/licenses/publisher-specific-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"},"sustainable_development_goals":[{"display_name":"Reduced inequalities","score":0.7029313445091248,"id":"https://metadata.un.org/sdg/10"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Large":[0],"language":[1,12],"models":[2],"(LLMs)":[3],"are":[4],"increasingly":[5],"applied":[6],"as":[7,164],"automatic":[8],"evaluators":[9],"for":[10,59,110],"natural":[11],"generation":[13],"assessment":[14],"often":[15],"using":[16],"pairwise":[17,122],"comparative":[18],"judgements.":[19],"Existing":[20],"approaches":[21],"typically":[22],"rely":[23],"on":[24,126],"single":[25],"judges":[26,30,37],"or":[27],"aggregate":[28],"multiple":[29],"assuming":[31],"equal":[32],"reliability.":[33,175],"In":[34],"practice,":[35],"LLM":[36,72,154],"vary":[38],"substantially":[39],"in":[40,71],"performance":[41],"across":[42],"tasks":[43],"and":[44,47,54,76,95,118,139],"evaluation":[45,129],"aspects,":[46],"their":[48],"judgment":[49],"probabilities":[50,74],"may":[51,62],"be":[52,63,162],"biased":[53],"inconsistent.":[55],"Furthermore,":[56],"human-labelled":[57],"supervision":[58],"judge":[60,112,119,174],"calibration":[61,167],"unavailable.":[64],"We":[65],"first":[66],"empirically":[67],"demonstrate":[68],"that":[69,78,105,132,140,159,169],"inconsistencies":[70],"comparison":[73],"exist":[75],"show":[77,131],"it":[79],"limits":[80],"the":[81,92,102,141,150],"effectiveness":[82],"of":[83,101,149,153],"direct":[84],"probability-based":[85],"ranking.":[86],"To":[87],"address":[88],"this,":[89],"we":[90],"study":[91],"LLM-asa-jury":[93],"setting":[94],"propose":[96],"BT-sigma,":[97],"a":[98,107],"judge-aware":[99],"extension":[100],"Bradley-Terry":[103],"model":[104],"introduces":[106],"discriminator":[108],"parameter":[109],"each":[111],"to":[113],"jointly":[114],"infer":[115],"item":[116],"rankings":[117],"reliability":[120],"from":[121],"comparisons":[123],"alone.":[124],"Experiments":[125],"benchmark":[127],"NLG":[128],"datasets":[130],"BT-sigma":[133,160],"consistently":[134],"outperforms":[135],"averaging-based":[136],"aggregation":[137,171],"methods,":[138],"learned":[142],"discriminators":[143],"strongly":[144],"correlate":[145],"with":[146],"independent":[147],"measures":[148],"cycle":[151],"consistency":[152],"judgments.":[155],"Further":[156],"analysis":[157],"reveals":[158],"can":[161],"interpreted":[163],"an":[165],"unsupervised":[166],"mechanism":[168],"improves":[170],"by":[172],"modelling":[173]},"counts_by_year":[],"updated_date":"2026-05-30T06:14:24.967023","created_date":"2026-02-20T00:00:00"}