{"id":"https://openalex.org/W7165140551","doi":"https://doi.org/10.48550/arxiv.2606.18709","title":"LLMs Struggle to Measure What Distinguishes Students of Different Proficiency Levels: A Study of Item Discrimination in Reading Comprehension Assessment","display_name":"LLMs Struggle to Measure What Distinguishes Students of Different Proficiency Levels: A Study of Item Discrimination in Reading Comprehension Assessment","publication_year":2026,"publication_date":"2026-06-17","ids":{"openalex":"https://openalex.org/W7165140551","doi":"https://doi.org/10.48550/arxiv.2606.18709"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.18709","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.18709","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.18709","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5138869328","display_name":"Han Chen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Han","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138930202","display_name":"Ming Li","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Ming","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138830622","display_name":"Chenguang Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Chenguang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5107311577","display_name":"Y Liang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liang, Yijun","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138907751","display_name":"Dawei Zhou","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhou, Dawei","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138905528","display_name":"Hong jiao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"jiao, Hong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5138927251","display_name":"Tianyi Zhou","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhou, Tianyi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10467","display_name":"Psychometric Methodologies and Testing","score":0.6427000164985657,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10467","display_name":"Psychometric Methodologies and Testing","score":0.6427000164985657,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11902","display_name":"Intelligent Tutoring Systems and Adaptive Learning","score":0.08860000222921371,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10959","display_name":"Student Assessment and Feedback","score":0.041099999099969864,"subfield":{"id":"https://openalex.org/subfields/3304","display_name":"Education"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/respondent","display_name":"Respondent","score":0.7584999799728394},{"id":"https://openalex.org/keywords/item-response-theory","display_name":"Item response theory","score":0.7034000158309937},{"id":"https://openalex.org/keywords/comprehension","display_name":"Comprehension","score":0.5539000034332275},{"id":"https://openalex.org/keywords/measure","display_name":"Measure (data warehouse)","score":0.5213000178337097},{"id":"https://openalex.org/keywords/correlation","display_name":"Correlation","score":0.5120000243186951},{"id":"https://openalex.org/keywords/test","display_name":"Test (biology)","score":0.4912000000476837},{"id":"https://openalex.org/keywords/reading-comprehension","display_name":"Reading comprehension","score":0.48089998960494995},{"id":"https://openalex.org/keywords/psychometrics","display_name":"Psychometrics","score":0.4602000117301941},{"id":"https://openalex.org/keywords/reading","display_name":"Reading (process)","score":0.4088999927043915}],"concepts":[{"id":"https://openalex.org/C2776640315","wikidata":"https://www.wikidata.org/wiki/Q7315941","display_name":"Respondent","level":2,"score":0.7584999799728394},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.7197999954223633},{"id":"https://openalex.org/C19875794","wikidata":"https://www.wikidata.org/wiki/Q1207340","display_name":"Item response theory","level":3,"score":0.7034000158309937},{"id":"https://openalex.org/C511192102","wikidata":"https://www.wikidata.org/wiki/Q5156948","display_name":"Comprehension","level":2,"score":0.5539000034332275},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.5213000178337097},{"id":"https://openalex.org/C117220453","wikidata":"https://www.wikidata.org/wiki/Q5172842","display_name":"Correlation","level":2,"score":0.5120000243186951},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.4912000000476837},{"id":"https://openalex.org/C2778780117","wikidata":"https://www.wikidata.org/wiki/Q3269423","display_name":"Reading comprehension","level":3,"score":0.48089998960494995},{"id":"https://openalex.org/C171606756","wikidata":"https://www.wikidata.org/wiki/Q506132","display_name":"Psychometrics","level":2,"score":0.4602000117301941},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.45669999718666077},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.420199990272522},{"id":"https://openalex.org/C554936623","wikidata":"https://www.wikidata.org/wiki/Q199657","display_name":"Reading (process)","level":2,"score":0.4088999927043915},{"id":"https://openalex.org/C116211729","wikidata":"https://www.wikidata.org/wiki/Q5372350","display_name":"Test validity","level":3,"score":0.3977000117301941},{"id":"https://openalex.org/C101266164","wikidata":"https://www.wikidata.org/wiki/Q2131821","display_name":"Rasch model","level":2,"score":0.37720000743865967},{"id":"https://openalex.org/C2778263599","wikidata":"https://www.wikidata.org/wiki/Q1669245","display_name":"Item analysis","level":3,"score":0.34360000491142273},{"id":"https://openalex.org/C2778143727","wikidata":"https://www.wikidata.org/wiki/Q1820650","display_name":"Readability","level":2,"score":0.3294999897480011},{"id":"https://openalex.org/C2778883600","wikidata":"https://www.wikidata.org/wiki/Q2390977","display_name":"Language proficiency","level":2,"score":0.32820001244544983},{"id":"https://openalex.org/C138496976","wikidata":"https://www.wikidata.org/wiki/Q175002","display_name":"Developmental psychology","level":1,"score":0.3174000084400177},{"id":"https://openalex.org/C181447626","wikidata":"https://www.wikidata.org/wiki/Q1224359","display_name":"Differential item functioning","level":4,"score":0.30640000104904175},{"id":"https://openalex.org/C20685875","wikidata":"https://www.wikidata.org/wiki/Q7239678","display_name":"Predictive validity","level":2,"score":0.29760000109672546},{"id":"https://openalex.org/C165838908","wikidata":"https://www.wikidata.org/wiki/Q736777","display_name":"Calibration","level":2,"score":0.289000004529953},{"id":"https://openalex.org/C188231857","wikidata":"https://www.wikidata.org/wiki/Q2585724","display_name":"Educational assessment","level":2,"score":0.27079999446868896},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.26489999890327454},{"id":"https://openalex.org/C2776291640","wikidata":"https://www.wikidata.org/wiki/Q2912517","display_name":"Value (mathematics)","level":2,"score":0.2621999979019165},{"id":"https://openalex.org/C166007726","wikidata":"https://www.wikidata.org/wiki/Q4536542","display_name":"Language assessment","level":2,"score":0.257999986410141},{"id":"https://openalex.org/C203151758","wikidata":"https://www.wikidata.org/wiki/Q3244907","display_name":"Standardized test","level":2,"score":0.2531999945640564}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.18709","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.18709","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.18709","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.18709","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"score":0.6523417830467224,"display_name":"Reduced inequalities","id":"https://metadata.un.org/sdg/10"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Item":[0],"discrimination":[1,68,76,99,149],"is":[2],"a":[3,118,127,140],"fundamental":[4],"psychometric":[5,156],"property":[6],"of":[7,121,143],"educational":[8],"assessment,":[9],"which":[10],"measures":[11],"whether":[12,32,44],"an":[13,74,151],"item":[14,39,48,148],"meaningfully":[15],"distinguishes":[16],"students":[17,22],"with":[18,23,110,133],"higher":[19],"proficiency":[20],"from":[21,78],"lower":[24],"proficiency.":[25],"While":[26],"various":[27],"existing":[28],"works":[29],"have":[30],"explored":[31],"large":[33],"language":[34],"models":[35,71],"(LLMs)":[36],"can":[37,46],"estimate":[38,73],"difficulty,":[40],"it":[41],"remains":[42],"unclear":[43],"they":[45,165],"capture":[47,170],"discrimination.":[49],"In":[50],"this":[51],"work,":[52],"we":[53],"evaluate":[54],"42":[55],"proprietary":[56],"and":[57,81],"open-weight":[58],"LLMs":[59,159],"in":[60],"zero-shot":[61],"settings":[62],"using":[63],"two":[64],"complementary":[65],"approaches:":[66],"direct":[67,105],"prediction,":[69],"where":[70,88],"explicitly":[72],"item's":[75],"value":[77],"its":[79],"content,":[80],"response-based":[82],"Classical":[83],"Test":[84],"Theory":[85],"(CTT)":[86],"calibration,":[87],"LLM":[89],"answers":[90],"are":[91],"treated":[92],"as":[93,150],"synthetic":[94,136],"student":[95],"responses":[96],"to":[97],"compute":[98],"scores.":[100],"Our":[101],"results":[102],"show":[103],"that":[104],"prediction":[106],"yields":[107],"weak":[108],"alignment":[109],"human-calibrated":[111],"discrimination:":[112],"the":[113,134],"best-performing":[114],"model":[115],"reaches":[116],"only":[117],"Spearman":[119,141],"correlation":[120,142],"0.152.":[122],"Response-based":[123],"CTT":[124],"calibration":[125],"provides":[126],"stronger":[128],"but":[129,164],"still":[130],"limited":[131],"signal,":[132,163],"all-persona":[135],"respondent":[137],"pool":[138],"reaching":[139],"0.241.":[144],"These":[145],"findings":[146],"highlight":[147],"open":[152],"challenge":[153],"for":[154],"LLM-based":[155],"evaluation:":[157],"current":[158],"contain":[160],"non-random":[161],"discrimination-relevant":[162],"do":[166],"not":[167],"yet":[168],"reliably":[169],"how":[171],"assessment":[172],"items":[173],"distinguish":[174],"human":[175],"students.":[176]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-06-19T00:00:00"}