{"id":"https://openalex.org/W7220954687","doi":"https://doi.org/10.48550/arxiv.2610.07755","title":"Trustworthy Method Comparison with AI Judges: Estimation and Design under Order, Batch, and Aggregation Effects","display_name":"Trustworthy Method Comparison with AI Judges: Estimation and Design under Order, Batch, and Aggregation Effects","publication_year":2026,"publication_date":"2026-10-06","ids":{"openalex":"https://openalex.org/W7220954687","doi":"https://doi.org/10.48550/arxiv.2610.07755"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2610.07755","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2610.07755","pdf_url":"https://arxiv.org/pdf/2610.07755","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2610.07755","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5077899199","display_name":"Tianxi Li","orcid":"https://orcid.org/0000-0003-4595-1777","observed_orcids":["https://orcid.org/0000-0003-4595-1777","https://orcid.org/0009-0009-8434-5156"]},"institutions":[{"id":"https://openalex.org/I130238516","display_name":"University of Minnesota","ror":"https://ror.org/017zqws13","country_code":"US","type":"education","lineage":["https://openalex.org/I130238516"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Li, Tianxi","raw_affiliation_strings":["School of Statistics , University of Minnesota"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Statistics , University of Minnesota","institution_ids":["https://openalex.org/I130238516"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5157901962","display_name":"Jie Ding","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I130238516","display_name":"University of Minnesota","ror":"https://ror.org/017zqws13","country_code":"US","type":"education","lineage":["https://openalex.org/I130238516"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ding, Jie","raw_affiliation_strings":["School of Statistics , University of Minnesota"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Statistics , University of Minnesota","institution_ids":["https://openalex.org/I130238516"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I130238516"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11798","display_name":"Optimal Experimental Design Methods","score":0.32249999046325684,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T11798","display_name":"Optimal Experimental Design Methods","score":0.32249999046325684,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10467","display_name":"Psychometric Methodologies and Testing","score":0.0658000037074089,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T13141","display_name":"Statistical Methods and Applications","score":0.06459999829530716,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/method-comparison","display_name":"method comparison","score":0.7319999933242798},{"id":"https://openalex.org/keywords/group-comparison","display_name":"group comparison","score":0.531000018119812},{"id":"https://openalex.org/keywords/order-effects","display_name":"order effects","score":0.546999990940094},{"id":"https://openalex.org/keywords/batch-effects","display_name":"batch effects","score":0.9490000009536743},{"id":"https://openalex.org/keywords/aggregation-effects","display_name":"aggregation effects","score":0.7979999780654907},{"id":"https://openalex.org/keywords/out-of-sample-forecasting","display_name":"out-of-sample forecasting","score":0.5419999957084656}],"concepts":[{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.6299999952316284},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5950000286102295},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.542900025844574},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5375999808311462},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.515500009059906},{"id":"https://openalex.org/C98763669","wikidata":"https://www.wikidata.org/wiki/Q176645","display_name":"Markov chain","level":2,"score":0.49480000138282776},{"id":"https://openalex.org/C81917197","wikidata":"https://www.wikidata.org/wiki/Q628760","display_name":"Selection (genetic algorithm)","level":2,"score":0.4693000018596649},{"id":"https://openalex.org/C2777212361","wikidata":"https://www.wikidata.org/wiki/Q5127848","display_name":"Class (philosophy)","level":2,"score":0.45680001378059387},{"id":"https://openalex.org/C163836022","wikidata":"https://www.wikidata.org/wiki/Q6771326","display_name":"Markov model","level":3,"score":0.4284999966621399},{"id":"https://openalex.org/C2778112365","wikidata":"https://www.wikidata.org/wiki/Q3511065","display_name":"Sequence (biology)","level":2,"score":0.4099000096321106},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3944000005722046},{"id":"https://openalex.org/C134261354","wikidata":"https://www.wikidata.org/wiki/Q938438","display_name":"Statistical inference","level":2,"score":0.38690000772476196},{"id":"https://openalex.org/C159886148","wikidata":"https://www.wikidata.org/wiki/Q176645","display_name":"Markov process","level":2,"score":0.3781000077724457},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.36419999599456787},{"id":"https://openalex.org/C120936955","wikidata":"https://www.wikidata.org/wiki/Q2155640","display_name":"Empirical research","level":2,"score":0.3578999936580658},{"id":"https://openalex.org/C153701036","wikidata":"https://www.wikidata.org/wiki/Q659974","display_name":"Trustworthiness","level":2,"score":0.34950000047683716},{"id":"https://openalex.org/C96250715","wikidata":"https://www.wikidata.org/wiki/Q965330","display_name":"Estimation","level":2,"score":0.34599998593330383},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.3334999978542328},{"id":"https://openalex.org/C93959086","wikidata":"https://www.wikidata.org/wiki/Q6888345","display_name":"Model selection","level":2,"score":0.3208000063896179},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.29269999265670776},{"id":"https://openalex.org/C23224414","wikidata":"https://www.wikidata.org/wiki/Q176769","display_name":"Hidden Markov model","level":2,"score":0.2808000147342682},{"id":"https://openalex.org/C111350023","wikidata":"https://www.wikidata.org/wiki/Q1191869","display_name":"Markov chain Monte Carlo","level":3,"score":0.2775000035762787},{"id":"https://openalex.org/C155846161","wikidata":"https://www.wikidata.org/wiki/Q1143367","display_name":"Graphical model","level":2,"score":0.27070000767707825},{"id":"https://openalex.org/C167928553","wikidata":"https://www.wikidata.org/wiki/Q1376021","display_name":"Estimation theory","level":2,"score":0.25459998846054077},{"id":"https://openalex.org/C106189395","wikidata":"https://www.wikidata.org/wiki/Q176789","display_name":"Markov decision process","level":3,"score":0.25279998779296875}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2610.07755","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2610.07755","pdf_url":"https://arxiv.org/pdf/2610.07755","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2610.07755","doi":"https://doi.org/10.48550/arxiv.2610.07755","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2610.07755","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2610.07755","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2610.07755","pdf_url":"https://arxiv.org/pdf/2610.07755","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G3011311721","display_name":"Collaborative Research: Statistical Modeling and Inference on Directed Network Data for Understanding Faculty Hiring Dynamics","funder_award_id":"2515367","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G3938928968","display_name":"CAREER: Continual Learning with Evolving Memory, Soft Supervision, and Cross-Domain Knowledge - Foundational Theory and Advanced Algorithms","funder_award_id":"2338506","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7220954687.pdf","grobid_xml":"https://content.openalex.org/works/W7220954687.grobid-xml"},"referenced_works_count":5,"referenced_works":["https://openalex.org/W4243004148","https://openalex.org/W4394708144","https://openalex.org/W4399694266","https://openalex.org/W4403885225","https://openalex.org/W7120088840"],"related_works":["https://openalex.org/W7126190339","https://openalex.org/W7214646924","https://openalex.org/W7123312012","https://openalex.org/W7214966144","https://openalex.org/W7203980511","https://openalex.org/W7138429202","https://openalex.org/W7167215375","https://openalex.org/W7154957730","https://openalex.org/W4415336032","https://openalex.org/W7130581838"],"abstract_inverted_index":{"Large":[0],"language":[1],"models":[2,49],"(LLMs)":[3],"are":[4,95],"increasingly":[5],"used":[6],"as":[7],"judges":[8,147],"for":[9],"automated":[10],"AI":[11,146],"evaluation.":[12],"A":[13],"common":[14],"practice":[15],"is":[16,77],"to":[17],"randomize":[18],"prompt":[19],"sequences":[20],"and":[21,69,84],"average":[22],"the":[23,113,121,124,129,140],"resulting":[24],"scores,":[25],"but":[26],"its":[27],"statistical":[28],"validity":[29,122],"remains":[30],"unclear.":[31],"We":[32,138],"show":[33],"that":[34],"LLM":[35],"evaluation":[36],"mechanisms":[37],"can":[38,89,102],"be":[39],"approximated":[40],"by":[41,52],"a":[42,61,80,85],"class":[43],"of":[44,112,123],"Markov":[45,63],"generalized":[46],"linear":[47],"mixed":[48],"(GLMMs),":[50],"supported":[51],"out-of-sample":[53],"predictions":[54],"across":[55],"three":[56],"major":[57],"commercial":[58],"LLMs.":[59],"Using":[60],"first-order":[62,130],"GLMM,":[64],"we":[65],"study":[66],"leaderboard":[67,73],"ranking":[68],"group":[70,98],"comparison.":[71],"For":[72,97],"ranking,":[74],"randomize-and-average":[75],"selection":[76],"consistent":[78],"under":[79],"mild":[81],"separation":[82],"condition,":[83],"Williams":[86],"square":[87],"design":[88],"improve":[90],"efficiency":[91],"when":[92],"item":[93],"qualities":[94],"close.":[96],"comparison,":[99],"naive":[100],"averaging":[101],"yield":[103],"inconsistent":[104],"conclusions":[105],"about":[106],"differences":[107],"in":[108,142],"group-level":[109],"quality":[110],"because":[111],"response":[114],"model's":[115],"nonlinearity.":[116],"Empirical":[117],"results":[118],"further":[119],"support":[120],"proposed":[125],"model-based":[126],"inference":[127],"beyond":[128],"theory,":[131],"including":[132],"settings":[133],"with":[134],"higher-order":[135],"sequence":[136],"memory.":[137],"illustrate":[139],"approach":[141],"an":[143],"application":[144],"where":[145],"compare":[148],"two":[149],"graphical":[150],"model":[151],"estimation":[152],"methods.":[153]},"counts_by_year":[],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2026-10-09T00:00:00"}