{"id":"https://openalex.org/W7169611917","doi":"https://doi.org/10.48550/arxiv.2607.15190","title":"Can We Trust Item Response Theory for AI Evaluation?","display_name":"Can We Trust Item Response Theory for AI Evaluation?","publication_year":2026,"publication_date":"2026-07-16","ids":{"openalex":"https://openalex.org/W7169611917","doi":"https://doi.org/10.48550/arxiv.2607.15190"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2607.15190","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2607.15190","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2607.15190","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5141081165","display_name":"Han Jiang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jiang, Han","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101377589","display_name":"Sunbeom Kwon","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kwon, Sunbeom","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5029995222","display_name":"Jun Luo","orcid":"https://orcid.org/0000-0003-3339-5566"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Luo, Jinwen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5141102505","display_name":"Ziang Xiao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xiao, Ziang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5141081504","display_name":"Susu Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Susu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10467","display_name":"Psychometric Methodologies and Testing","score":0.7803000211715698,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10467","display_name":"Psychometric Methodologies and Testing","score":0.7803000211715698,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11577","display_name":"Cognitive Abilities and Testing","score":0.04820000007748604,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T13283","display_name":"Mental Health Research Topics","score":0.017400000244379044,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/item-response-theory","display_name":"Item response theory","score":0.7143999934196472},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.6394000053405762},{"id":"https://openalex.org/keywords/estimator","display_name":"Estimator","score":0.6297000050544739},{"id":"https://openalex.org/keywords/benchmarking","display_name":"Benchmarking","score":0.593999981880188},{"id":"https://openalex.org/keywords/markov-chain-monte-carlo","display_name":"Markov chain Monte Carlo","score":0.527999997138977},{"id":"https://openalex.org/keywords/leverage","display_name":"Leverage (statistics)","score":0.5152000188827515},{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.46799999475479126},{"id":"https://openalex.org/keywords/ranking","display_name":"Ranking (information retrieval)","score":0.37940001487731934},{"id":"https://openalex.org/keywords/statistical-model","display_name":"Statistical model","score":0.35679998993873596},{"id":"https://openalex.org/keywords/markov-chain","display_name":"Markov chain","score":0.35580000281333923}],"concepts":[{"id":"https://openalex.org/C19875794","wikidata":"https://www.wikidata.org/wiki/Q1207340","display_name":"Item response theory","level":3,"score":0.7143999934196472},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6610000133514404},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.6394000053405762},{"id":"https://openalex.org/C185429906","wikidata":"https://www.wikidata.org/wiki/Q1130160","display_name":"Estimator","level":2,"score":0.6297000050544739},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.595300018787384},{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.593999981880188},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5288000106811523},{"id":"https://openalex.org/C111350023","wikidata":"https://www.wikidata.org/wiki/Q1191869","display_name":"Markov chain Monte Carlo","level":3,"score":0.527999997138977},{"id":"https://openalex.org/C153083717","wikidata":"https://www.wikidata.org/wiki/Q6535263","display_name":"Leverage (statistics)","level":2,"score":0.5152000188827515},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.46799999475479126},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.4530999958515167},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.3919000029563904},{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.37940001487731934},{"id":"https://openalex.org/C114289077","wikidata":"https://www.wikidata.org/wiki/Q3284399","display_name":"Statistical model","level":2,"score":0.35679998993873596},{"id":"https://openalex.org/C98763669","wikidata":"https://www.wikidata.org/wiki/Q176645","display_name":"Markov chain","level":2,"score":0.35580000281333923},{"id":"https://openalex.org/C102366305","wikidata":"https://www.wikidata.org/wiki/Q1097688","display_name":"Nonparametric statistics","level":2,"score":0.33180001378059387},{"id":"https://openalex.org/C79337645","wikidata":"https://www.wikidata.org/wiki/Q779824","display_name":"Outlier","level":2,"score":0.3192000091075897},{"id":"https://openalex.org/C19499675","wikidata":"https://www.wikidata.org/wiki/Q232207","display_name":"Monte Carlo method","level":2,"score":0.31470000743865967},{"id":"https://openalex.org/C150921843","wikidata":"https://www.wikidata.org/wiki/Q1170431","display_name":"Resampling","level":2,"score":0.3142000138759613},{"id":"https://openalex.org/C198531522","wikidata":"https://www.wikidata.org/wiki/Q485146","display_name":"Sample (material)","level":2,"score":0.3018999993801117},{"id":"https://openalex.org/C67186912","wikidata":"https://www.wikidata.org/wiki/Q367664","display_name":"Data modeling","level":2,"score":0.2980000078678131},{"id":"https://openalex.org/C153701036","wikidata":"https://www.wikidata.org/wiki/Q659974","display_name":"Trustworthiness","level":2,"score":0.29420000314712524},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.2939999997615814},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.2928999960422516},{"id":"https://openalex.org/C129848803","wikidata":"https://www.wikidata.org/wiki/Q2564360","display_name":"Sample size determination","level":2,"score":0.2865000069141388},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.2824999988079071},{"id":"https://openalex.org/C167966045","wikidata":"https://www.wikidata.org/wiki/Q5532625","display_name":"Generative model","level":3,"score":0.27880001068115234},{"id":"https://openalex.org/C164226766","wikidata":"https://www.wikidata.org/wiki/Q7293202","display_name":"Rank (graph theory)","level":2,"score":0.2736000120639801},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.2703999876976013},{"id":"https://openalex.org/C34736171","wikidata":"https://www.wikidata.org/wiki/Q918333","display_name":"Preprocessor","level":2,"score":0.26570001244544983},{"id":"https://openalex.org/C107673813","wikidata":"https://www.wikidata.org/wiki/Q812534","display_name":"Bayesian probability","level":2,"score":0.2637999951839447},{"id":"https://openalex.org/C144986985","wikidata":"https://www.wikidata.org/wiki/Q871236","display_name":"Hierarchical database model","level":2,"score":0.2567000091075897},{"id":"https://openalex.org/C134261354","wikidata":"https://www.wikidata.org/wiki/Q938438","display_name":"Statistical inference","level":2,"score":0.25270000100135803},{"id":"https://openalex.org/C159886148","wikidata":"https://www.wikidata.org/wiki/Q176645","display_name":"Markov process","level":2,"score":0.251800000667572}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2607.15190","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2607.15190","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2607.15190","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2607.15190","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"AI":[0,26,79,192],"benchmarks":[1,47],"increasingly":[2],"leverage":[3],"item-level":[4,169],"statistical":[5],"models,":[6,52],"particularly":[7],"item":[8,82,149],"response":[9,96],"theory":[10],"(IRT),":[11],"to":[12],"estimate":[13],"model":[14,144,178],"capabilities,":[15],"rank":[16],"systems,":[17],"select":[18],"informative":[19],"examples,":[20],"and":[21,56,84,103,122,137,148,170,195,199],"diagnose":[22],"benchmark":[23,27,111,161],"quality.":[24],"However,":[25],"data":[28,33],"often":[29],"departs":[30],"from":[31,88],"the":[32,73,138],"regime":[34,70],"of":[35,75,140],"human":[36],"testing,":[37],"for":[38,78,203],"which":[39],"standard":[40],"IRT":[41,76,101,141],"estimation":[42,106],"tools":[43,107],"were":[44],"originally":[45],"developed:":[46],"typically":[48],"involve":[49],"fewer":[50],"evaluated":[51],"far":[53],"more":[54],"items,":[55],"capability":[57,85],"distributions":[58,86],"that":[59,153],"may":[60],"be":[61],"skewed,":[62],"clustered,":[63],"or":[64,175,189],"multimodal.":[65],"We":[66],"examine":[67],"how":[68],"these":[69],"mismatches":[71],"challenge":[72],"reliability":[74,139],"modeling":[77],"evaluation.":[80],"Using":[81],"parameters":[83],"derived":[87],"six":[89],"widely":[90],"used":[91,108],"LLM":[92],"benchmarks,":[93],"we":[94,131],"simulate":[95],"matrices":[97],"under":[98],"three":[99],"common":[100],"models":[102,186],"compare":[104],"four":[105],"in":[109,159],"recent":[110],"studies:":[112],"marginal":[113],"maximum":[114],"likelihood,":[115],"Markov":[116],"chain":[117],"Monte":[118],"Carlo,":[119],"variational":[120],"inference,":[121],"a":[123],"neural":[124],"pseudo-Siamese":[125],"estimator.":[126],"Across":[127],"18,000":[128],"simulation":[129],"conditions,":[130],"systematically":[132],"evaluate":[133],"computational":[134],"feasibility,":[135],"scalability,":[136],"inferences":[142,172],"about":[143],"rankings,":[145],"predicted":[146],"performance,":[147],"characteristics.":[150],"Results":[151],"show":[152],"classical":[154],"estimators":[155,165],"can":[156,166],"become":[157],"infeasible":[158],"large":[160],"settings,":[162],"whereas":[163],"scalable":[164],"produce":[167],"unreliable":[168],"ranking":[171],"with":[173],"small":[174],"non-normally":[176],"distributed":[177],"sets.":[179],"This":[180],"study":[181],"identifies":[182],"when":[183],"latent":[184],"trait":[185],"reliably":[187],"support":[188],"risk":[190],"distorting":[191],"benchmarking":[193],"claims,":[194],"what":[196],"sample":[197],"sizes":[198],"diagnostics":[200],"are":[201],"needed":[202],"trustworthy":[204],"use.":[205]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-07-18T00:00:00"}