{"id":"https://openalex.org/W7204157040","doi":"https://doi.org/10.48550/arxiv.2608.22295","title":"LLM Evaluation on Unseen Questions: Contextual Multidimensional IRT Model","display_name":"LLM Evaluation on Unseen Questions: Contextual Multidimensional IRT Model","publication_year":2026,"publication_date":"2026-08-23","ids":{"openalex":"https://openalex.org/W7204157040","doi":"https://doi.org/10.48550/arxiv.2608.22295"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2608.22295","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2608.22295","pdf_url":"https://arxiv.org/pdf/2608.22295","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2608.22295","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5121328482","display_name":"Ergan Shang","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Shang, Ergan","raw_affiliation_strings":["Department of Statistics and Data Science , Carnegie Mel- lon University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Statistics and Data Science , Carnegie Mel- lon University","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138201655","display_name":"Weijing Tang","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Tang, Weijing","raw_affiliation_strings":["Department of Statistics and Data Science , Carnegie Mel- lon University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Statistics and Data Science , Carnegie Mel- lon University","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5074500746","display_name":"Yinqiu He","orcid":"https://orcid.org/0000-0002-4829-033X","observed_orcids":["https://orcid.org/0000-0002-4829-033X"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"He, Yinqiu","raw_affiliation_strings":["Department of Statistics , University of Madison"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Statistics , University of Madison","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10467","display_name":"Psychometric Methodologies and Testing","score":0.4000999927520752,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10467","display_name":"Psychometric Methodologies and Testing","score":0.4000999927520752,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.391400009393692,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.0215000007301569,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/llm-evaluation","display_name":"LLM evaluation","score":0.9729999899864197},{"id":"https://openalex.org/keywords/multidimensional-item-response-theory","display_name":"multidimensional item response theory","score":0.9860000014305115},{"id":"https://openalex.org/keywords/question-embedding","display_name":"question embedding","score":0.890999972820282},{"id":"https://openalex.org/keywords/cross-scene-generalization","display_name":"cross-scene generalization","score":0.878000020980835},{"id":"https://openalex.org/keywords/psychometric-modeling","display_name":"psychometric modeling","score":0.5979999899864197}],"concepts":[{"id":"https://openalex.org/C27158222","wikidata":"https://www.wikidata.org/wiki/Q5532422","display_name":"Generalizability theory","level":2,"score":0.8411999940872192},{"id":"https://openalex.org/C177148314","wikidata":"https://www.wikidata.org/wiki/Q170084","display_name":"Generalization","level":2,"score":0.6815999746322632},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6642000079154968},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6193000078201294},{"id":"https://openalex.org/C19875794","wikidata":"https://www.wikidata.org/wiki/Q1207340","display_name":"Item response theory","level":3,"score":0.5687000155448914},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.52920001745224},{"id":"https://openalex.org/C2778334786","wikidata":"https://www.wikidata.org/wiki/Q1586270","display_name":"Variation (astronomy)","level":2,"score":0.5166000127792358},{"id":"https://openalex.org/C2780586882","wikidata":"https://www.wikidata.org/wiki/Q7520643","display_name":"Simple (philosophy)","level":2,"score":0.430400013923645},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.4083999991416931},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3540000021457672},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.3409000039100647},{"id":"https://openalex.org/C198531522","wikidata":"https://www.wikidata.org/wiki/Q485146","display_name":"Sample (material)","level":2,"score":0.33799999952316284},{"id":"https://openalex.org/C51167844","wikidata":"https://www.wikidata.org/wiki/Q4422623","display_name":"Latent variable","level":2,"score":0.30799999833106995},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.30000001192092896},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.2919999957084656},{"id":"https://openalex.org/C3019022308","wikidata":"https://www.wikidata.org/wiki/Q1418353","display_name":"Multidimensional data","level":2,"score":0.29010000824928284},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.27970001101493835},{"id":"https://openalex.org/C136197465","wikidata":"https://www.wikidata.org/wiki/Q1729295","display_name":"Variety (cybernetics)","level":2,"score":0.2750000059604645},{"id":"https://openalex.org/C67186912","wikidata":"https://www.wikidata.org/wiki/Q367664","display_name":"Data modeling","level":2,"score":0.2728999853134155},{"id":"https://openalex.org/C2780801425","wikidata":"https://www.wikidata.org/wiki/Q5164392","display_name":"Construct (python library)","level":2,"score":0.26930001378059387},{"id":"https://openalex.org/C168167062","wikidata":"https://www.wikidata.org/wiki/Q1117970","display_name":"Component (thermodynamics)","level":2,"score":0.2667999863624573},{"id":"https://openalex.org/C171606756","wikidata":"https://www.wikidata.org/wiki/Q506132","display_name":"Psychometrics","level":2,"score":0.25}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2608.22295","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2608.22295","pdf_url":"https://arxiv.org/pdf/2608.22295","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2608.22295","doi":"https://doi.org/10.48550/arxiv.2608.22295","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2608.22295","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2608.22295","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2608.22295","pdf_url":"https://arxiv.org/pdf/2608.22295","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7204157040.pdf","grobid_xml":"https://content.openalex.org/works/W7204157040.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4415331690","https://openalex.org/W7138005650","https://openalex.org/W7223417875","https://openalex.org/W4415590663","https://openalex.org/W4416373968","https://openalex.org/W7166884377","https://openalex.org/W7154483052","https://openalex.org/W7128365992","https://openalex.org/W7212087654","https://openalex.org/W7163695570"],"abstract_inverted_index":{"Evaluation":[0],"of":[1,23,126],"large":[2,21],"language":[3],"models":[4],"(LLMs)":[5],"increasingly":[6],"requires":[7],"predicting":[8],"how":[9],"a":[10,56,123,163,178],"model":[11,46,66],"will":[12],"perform":[13],"on":[14,74],"new":[15,24],"questions":[16],"or":[17],"tasks":[18],"before":[19],"collecting":[20],"amounts":[22],"annotations.":[25],"This":[26],"problem":[27],"is":[28,162],"challenging":[29],"because":[30],"question":[31,68,87,109],"difficulty,":[32],"scenario,":[33],"and":[34,117,168],"underlying":[35],"capability":[36,83,127],"demands":[37],"can":[38],"vary":[39],"substantially.":[40],"Simple":[41],"retrospective":[42],"averages":[43],"may":[44],"confound":[45],"ability":[47],"with":[48,67],"item":[49,63,91],"characteristics.":[50],"In":[51],"this":[52],"paper,":[53],"we":[54,102],"study":[55],"model-based":[57],"evaluation":[58],"framework":[59,78],"that":[60,104,118,142,158],"combines":[61],"multidimensional":[62,119],"response":[64],"theory":[65],"contexts":[69],"to":[70,89,95,114],"predict":[71],"LLM":[72,170],"performance":[73],"unseen":[75],"questions.":[76],"The":[77],"represents":[79],"LLMs":[80],"through":[81],"latent":[82,120],"profiles":[84],"while":[85,172],"using":[86],"content":[88],"inform":[90],"characteristics,":[92],"allowing":[93],"information":[94],"transfer":[96],"beyond":[97],"previously":[98],"observed":[99],"items.":[100],"Empirically,":[101],"find":[103],"for":[105,166],"within-scenario":[106],"evaluation,":[107,171],"incorporating":[108],"embeddings":[110],"improves":[111],"prediction":[112,151],"relative":[113],"model-free":[115],"baselines,":[116],"structure":[121],"provides":[122],"richer":[124],"description":[125],"variation":[128],"than":[129],"unidimensional":[130],"alternatives.":[131],"At":[132],"the":[133,143],"same":[134],"time,":[135],"our":[136],"results":[137],"reveal":[138],"an":[139],"important":[140],"limitation":[141],"generalizability":[144],"does":[145],"not":[146],"necessarily":[147],"translate":[148],"into":[149],"reliable":[150],"under":[152],"cross-scenario":[153,175],"shift.":[154],"These":[155],"findings":[156],"suggest":[157],"context-aware":[159],"psychometric":[160],"modeling":[161],"promising":[164],"direction":[165],"efficient":[167],"interpretable":[169],"also":[173],"highlighting":[174],"generalization":[176],"as":[177],"central":[179],"open":[180],"challenge.":[181]},"counts_by_year":[],"updated_date":"2026-10-28T10:33:24.094632","created_date":"2026-08-26T00:00:00"}