{"id":"https://openalex.org/W7151265763","doi":"https://doi.org/10.48550/arxiv.2604.03742","title":"Structured Multi-Criteria Evaluation of Large Language Models with Fuzzy Analytic Hierarchy Process and DualJudge","display_name":"Structured Multi-Criteria Evaluation of Large Language Models with Fuzzy Analytic Hierarchy Process and DualJudge","publication_year":2026,"publication_date":"2026-04-04","ids":{"openalex":"https://openalex.org/W7151265763","doi":"https://doi.org/10.48550/arxiv.2604.03742"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2604.03742","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.03742","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2604.03742","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5133081718","display_name":"Yulong He","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"He, Yulong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133116568","display_name":"Ivan Smirnov","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Smirnov, Ivan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133138085","display_name":"Dmitry Fedrushkov","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Fedrushkov, Dmitry","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5029904389","display_name":"Sergey V. Kovalchuk","orcid":"https://orcid.org/0000-0001-8828-4615"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kovalchuk, Sergey","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5093593891","display_name":"Ilya Revin","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Revin, Ilya","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.2556000053882599,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.2556000053882599,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.10050000250339508,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.060499999672174454,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/fuzzy-logic","display_name":"Fuzzy logic","score":0.6144999861717224},{"id":"https://openalex.org/keywords/analytic-hierarchy-process","display_name":"Analytic hierarchy process","score":0.5031999945640564},{"id":"https://openalex.org/keywords/stability","display_name":"Stability (learning theory)","score":0.4422999918460846},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.4343999922275543},{"id":"https://openalex.org/keywords/hierarchy","display_name":"Hierarchy","score":0.41499999165534973}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6290000081062317},{"id":"https://openalex.org/C58166","wikidata":"https://www.wikidata.org/wiki/Q224821","display_name":"Fuzzy logic","level":2,"score":0.6144999861717224},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5482000112533569},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5138999819755554},{"id":"https://openalex.org/C87345402","wikidata":"https://www.wikidata.org/wiki/Q485202","display_name":"Analytic hierarchy process","level":2,"score":0.5031999945640564},{"id":"https://openalex.org/C112972136","wikidata":"https://www.wikidata.org/wiki/Q7595718","display_name":"Stability (learning theory)","level":2,"score":0.4422999918460846},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.4343999922275543},{"id":"https://openalex.org/C31170391","wikidata":"https://www.wikidata.org/wiki/Q188619","display_name":"Hierarchy","level":2,"score":0.41499999165534973},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.390500009059906},{"id":"https://openalex.org/C2778029271","wikidata":"https://www.wikidata.org/wiki/Q5421931","display_name":"Extension (predicate logic)","level":2,"score":0.3752000033855438},{"id":"https://openalex.org/C42011625","wikidata":"https://www.wikidata.org/wiki/Q1055058","display_name":"Fuzzy set","level":3,"score":0.35679998993873596},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.28110000491142273}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2604.03742","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.03742","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2604.03742","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.03742","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/16","score":0.4603513181209564,"display_name":"Peace, Justice and strong institutions"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Effective":[0],"evaluation":[1,33,144],"of":[2,140],"large":[3],"language":[4],"models":[5,45],"(LLMs)":[6],"remains":[7],"a":[8,38,112,153],"critical":[9],"bottleneck,":[10],"as":[11,152],"conventional":[12],"direct":[13,88,123],"scoring":[14,89],"often":[15],"yields":[16],"inconsistent":[17],"and":[18,69,83,93,142],"opaque":[19],"judgments.":[20,76],"In":[21],"this":[22],"work,":[23],"we":[24,109],"adapt":[25],"the":[26,137],"Analytic":[27],"Hierarchy":[28],"Process":[29],"(AHP)":[30],"to":[31],"LLM-based":[32],"and,":[34],"more":[35,74,157],"importantly,":[36],"propose":[37,110],"confidence-aware":[39],"Fuzzy":[40],"AHP":[41,85,127],"(FAHP)":[42],"extension":[43],"that":[44,80,119],"epistemic":[46],"uncertainty":[47],"via":[48,129],"triangular":[49],"fuzzy":[50,84],"numbers":[51],"modulated":[52],"by":[53,116],"LLM-generated":[54],"confidence":[55],"scores.":[56],"Systematically":[57],"validated":[58],"on":[59,106],"JudgeBench,":[60],"our":[61],"structured":[62,126,150],"approach":[63],"decomposes":[64],"assessments":[65],"into":[66],"explicit":[67],"criteria":[68],"incorporates":[70],"uncertainty-aware":[71,149],"aggregation,":[72],"producing":[73],"calibrated":[75],"Extensive":[77],"experiments":[78],"demonstrate":[79],"both":[81],"crisp":[82],"consistently":[86],"outperform":[87],"across":[90],"model":[91],"scales":[92],"dataset":[94],"splits,":[95],"with":[96,125],"FAHP":[97],"showing":[98],"superior":[99],"stability":[100],"in":[101],"uncertain":[102],"comparison":[103],"scenarios.":[104],"Building":[105],"these":[107],"insights,":[108],"\\textbf{DualJudge},":[111],"hybrid":[113],"framework":[114],"inspired":[115],"Dual-Process":[117],"Theory":[118],"adaptively":[120],"fuses":[121],"holistic":[122],"scores":[124],"outputs":[128],"consistency-aware":[130],"weighting.":[131],"DualJudge":[132],"achieves":[133],"state-of-the-art":[134],"performance,":[135],"underscoring":[136],"complementary":[138],"strengths":[139],"intuitive":[141],"deliberative":[143],"paradigms.":[145],"These":[146],"results":[147],"establish":[148],"reasoning":[151],"principled":[154],"pathway":[155],"toward":[156],"reliable":[158],"LLM":[159],"assessment.":[160],"Code":[161],"is":[162],"available":[163],"at":[164],"https://github.com/hreyulog/AHP_llm_judge.":[165]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-04-08T00:00:00"}