{"id":"https://openalex.org/W7127935417","doi":"https://doi.org/10.48550/arxiv.2602.05110","title":"Understanding LLM Evaluator Behavior: A Structured Multi-Evaluator Framework for Merchant Risk Assessment","display_name":"Understanding LLM Evaluator Behavior: A Structured Multi-Evaluator Framework for Merchant Risk Assessment","publication_year":2026,"publication_date":"2026-02-04","ids":{"openalex":"https://openalex.org/W7127935417","doi":"https://doi.org/10.48550/arxiv.2602.05110"},"language":null,"primary_location":{"id":"pmh:doi:10.48550/arxiv.2602.05110","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4406922384","display_name":"Open MIND","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":null,"any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5125122055","display_name":"Liang Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Liang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5125204339","display_name":"Junpeng Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Junpeng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5011045620","display_name":"Chin\u2010Chia Michael Yeh","orcid":"https://orcid.org/0000-0002-9807-2963"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yeh, Chin-chia Michael","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5125212110","display_name":"Yan Zheng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zheng, Yan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5125250527","display_name":"Jiarui Sun","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sun, Jiarui","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5125198503","display_name":"Xiran Fan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Fan, Xiran","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100728265","display_name":"Xin Dai","orcid":"https://orcid.org/0000-0002-2622-7575"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Dai, Xin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101194694","display_name":"Yujie Fan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Fan, Yujie","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5125201821","display_name":"Yiwei Cai","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Cai, Yiwei","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.15469999611377716,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.15469999611377716,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11995","display_name":"FinTech, Crowdfunding, Digital Finance","score":0.06800000369548798,"subfield":{"id":"https://openalex.org/subfields/1404","display_name":"Management Information Systems"},"field":{"id":"https://openalex.org/fields/14","display_name":"Business, Management and Accounting"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.052799999713897705,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/rubric","display_name":"Rubric","score":0.8202000260353088},{"id":"https://openalex.org/keywords/metric","display_name":"Metric (unit)","score":0.613099992275238},{"id":"https://openalex.org/keywords/workflow","display_name":"Workflow","score":0.6093999743461609},{"id":"https://openalex.org/keywords/risk-assessment","display_name":"Risk assessment","score":0.4683000147342682},{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.4659000039100647},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.4555000066757202}],"concepts":[{"id":"https://openalex.org/C111640148","wikidata":"https://www.wikidata.org/wiki/Q847349","display_name":"Rubric","level":2,"score":0.8202000260353088},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.613099992275238},{"id":"https://openalex.org/C177212765","wikidata":"https://www.wikidata.org/wiki/Q627335","display_name":"Workflow","level":2,"score":0.6093999743461609},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5121999979019165},{"id":"https://openalex.org/C12174686","wikidata":"https://www.wikidata.org/wiki/Q1058438","display_name":"Risk assessment","level":2,"score":0.4683000147342682},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.4659000039100647},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.4555000066757202},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.42590001225471497},{"id":"https://openalex.org/C103278499","wikidata":"https://www.wikidata.org/wiki/Q254465","display_name":"Similarity (geometry)","level":3,"score":0.3456000089645386},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3433000147342682},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.329800009727478},{"id":"https://openalex.org/C112930515","wikidata":"https://www.wikidata.org/wiki/Q4389547","display_name":"Risk analysis (engineering)","level":1,"score":0.3093000054359436},{"id":"https://openalex.org/C38721330","wikidata":"https://www.wikidata.org/wiki/Q80056","display_name":"Sonnet","level":3,"score":0.3041999936103821},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3019999861717224},{"id":"https://openalex.org/C150241267","wikidata":"https://www.wikidata.org/wiki/Q7336285","display_name":"Risk management tools","level":3,"score":0.2662999927997589},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.2632000148296356},{"id":"https://openalex.org/C32896092","wikidata":"https://www.wikidata.org/wiki/Q189447","display_name":"Risk management","level":2,"score":0.25929999351501465},{"id":"https://openalex.org/C162118730","wikidata":"https://www.wikidata.org/wiki/Q1128453","display_name":"Actuarial science","level":1,"score":0.25429999828338623},{"id":"https://openalex.org/C2776436953","wikidata":"https://www.wikidata.org/wiki/Q5163215","display_name":"Consistency (knowledge bases)","level":2,"score":0.2513999938964844}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:doi:10.48550/arxiv.2602.05110","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4406922384","display_name":"Open MIND","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"},{"id":"doi:10.48550/arxiv.2602.05110","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2602.05110","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:doi:10.48550/arxiv.2602.05110","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4406922384","display_name":"Open MIND","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Large":[0],"Language":[1],"Models":[2],"(LLMs)":[3],"are":[4],"increasingly":[5],"used":[6],"as":[7],"evaluators":[8],"of":[9,90,99,159],"reasoning":[10,32],"quality,":[11],"yet":[12],"their":[13],"reliability":[14],"and":[15,52,59,66,101,109,122,154,161,210],"bias":[16,116,127,131,158],"in":[17,33,207,217],"payments-risk":[18],"settings":[19],"remain":[20],"poorly":[21],"understood.":[22],"We":[23],"introduce":[24,75],"a":[25,42,71,76,95,200],"structured":[26],"multi-evaluator":[27],"framework":[28,192,198],"for":[29,203,214],"assessing":[30],"LLM":[31,144],"Merchant":[34],"Category":[35],"Code":[36],"(MCC)-based":[37],"merchant":[38],"risk":[39,62],"assessment,":[40],"combining":[41],"five-criterion":[43],"rubric":[44],"with":[45,130,168],"Monte-Carlo":[46],"scoring":[47],"to":[48,87,187],"evaluate":[49],"rationale":[50],"quality":[51],"evaluator":[53],"stability.":[54],"Five":[55],"frontier":[56],"LLMs":[57],"generate":[58],"cross-evaluate":[60],"MCC":[61],"rationales":[63],"under":[64,136],"attributed":[65],"anonymized":[67],"conditions.":[68],"To":[69],"establish":[70],"judge-independent":[72],"reference,":[73],"we":[74],"consensus-deviation":[77],"metric":[78],"that":[79,155,190],"eliminates":[80],"circularity":[81],"by":[82,133,139],"comparing":[83],"each":[84],"judge's":[85],"score":[86],"the":[88,156,191,197,212],"mean":[89],"all":[91],"other":[92],"judges,":[93],"yielding":[94],"theoretically":[96],"grounded":[97],"measure":[98],"self-evaluation":[100,115],"cross-model":[102],"deviation.":[103],"Results":[104],"reveal":[105],"substantial":[106],"heterogeneity:":[107],"GPT-5.1":[108,160],"Claude":[110,162],"4.5":[111,163],"Sonnet":[112,164],"show":[113],"negative":[114,157],"(-0.33,":[117],"-0.31),":[118],"while":[119],"Gemini-2.5":[120],"Pro":[121],"Grok":[123],"4":[124],"display":[125],"positive":[126],"(+0.77,":[128],"+0.71),":[129],"attenuating":[132],"25.8":[134],"percent":[135],"anonymization.":[137],"Evaluation":[138],"26":[140],"payment-industry":[141],"experts":[142],"shows":[143,176],"judges":[145],"assign":[146],"scores":[147],"averaging":[148],"+0.46":[149],"points":[150],"above":[151],"human":[152,169],"consensus,":[153],"reflects":[165],"closer":[166],"alignment":[167,182],"judgment.":[170],"Ground-truth":[171],"validation":[172],"using":[173],"payment-network":[174],"data":[175],"four":[177],"models":[178],"exhibit":[179],"statistically":[180],"significant":[181],"(Spearman":[183],"rho":[184],"=":[185],"0.56":[186],"0.77),":[188],"confirming":[189],"captures":[193],"genuine":[194],"quality.":[195],"Overall,":[196],"provides":[199],"replicable":[201],"basis":[202],"evaluating":[204],"LLM-as-a-judge":[205],"systems":[206],"payment-risk":[208],"workflows":[209],"highlights":[211],"need":[213],"bias-aware":[215],"protocols":[216],"operational":[218],"financial":[219],"settings.":[220]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-02-07T00:00:00"}