{"id":"https://openalex.org/W7160014530","doi":"https://doi.org/10.48550/arxiv.2605.00238","title":"Estimating LLM Grading Ability and Response Difficulty in Automatic Short Answer Grading via Item Response Theory","display_name":"Estimating LLM Grading Ability and Response Difficulty in Automatic Short Answer Grading via Item Response Theory","publication_year":2026,"publication_date":"2026-04-30","ids":{"openalex":"https://openalex.org/W7160014530","doi":"https://doi.org/10.48550/arxiv.2605.00238"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.00238","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.00238","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.00238","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5073181719","display_name":"Longwei Cong","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Cong, Longwei","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5089202000","display_name":"Sonja Hahn","orcid":"https://orcid.org/0000-0002-5461-6383"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hahn, Sonja","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5054545684","display_name":"Sebastian Gombert","orcid":"https://orcid.org/0000-0001-5598-9547"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gombert, Sebastian","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5013020856","display_name":"Leon Camus","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Camus, Leon","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135217289","display_name":"Hendrik Drachsler","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Drachsler, Hendrik","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5011679572","display_name":"Ulf Kroehne","orcid":"https://orcid.org/0000-0002-0412-169X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kroehne, Ulf","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11902","display_name":"Intelligent Tutoring Systems and Adaptive Learning","score":0.3433000147342682,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11902","display_name":"Intelligent Tutoring Systems and Adaptive Learning","score":0.3433000147342682,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.3179999887943268,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11587","display_name":"Second Language Acquisition and Learning","score":0.05660000070929527,"subfield":{"id":"https://openalex.org/subfields/3204","display_name":"Developmental and Educational Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/grading","display_name":"Grading (engineering)","score":0.7724000215530396},{"id":"https://openalex.org/keywords/correctness","display_name":"Correctness","score":0.5371000170707703},{"id":"https://openalex.org/keywords/item-response-theory","display_name":"Item response theory","score":0.4375999867916107},{"id":"https://openalex.org/keywords/robustness","display_name":"Robustness (evolution)","score":0.4348999857902527},{"id":"https://openalex.org/keywords/confusion","display_name":"Confusion","score":0.4172999858856201}],"concepts":[{"id":"https://openalex.org/C2777286243","wikidata":"https://www.wikidata.org/wiki/Q5591926","display_name":"Grading (engineering)","level":2,"score":0.7724000215530396},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5805000066757202},{"id":"https://openalex.org/C55439883","wikidata":"https://www.wikidata.org/wiki/Q360812","display_name":"Correctness","level":2,"score":0.5371000170707703},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.5238999724388123},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5006999969482422},{"id":"https://openalex.org/C19875794","wikidata":"https://www.wikidata.org/wiki/Q1207340","display_name":"Item response theory","level":3,"score":0.4375999867916107},{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.4348999857902527},{"id":"https://openalex.org/C2781140086","wikidata":"https://www.wikidata.org/wiki/Q557945","display_name":"Confusion","level":2,"score":0.4172999858856201},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.38679999113082886},{"id":"https://openalex.org/C191399111","wikidata":"https://www.wikidata.org/wiki/Q64861","display_name":"Transitive relation","level":2,"score":0.3276999890804291},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.2939999997615814},{"id":"https://openalex.org/C170133592","wikidata":"https://www.wikidata.org/wiki/Q1806883","display_name":"Latent semantic analysis","level":2,"score":0.2743000090122223},{"id":"https://openalex.org/C41608201","wikidata":"https://www.wikidata.org/wiki/Q980509","display_name":"Embedding","level":2,"score":0.2556999921798706},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.2524999976158142}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.00238","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.00238","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.00238","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.00238","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"score":0.7355982661247253,"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Automated":[0],"short":[1],"answer":[2],"grading":[3,30,38,56,67,123],"(ASAG)":[4],"with":[5,13,113,175],"large":[6],"language":[7],"models":[8,55,112],"(LLMs)":[9],"is":[10,173],"commonly":[11],"evaluated":[12],"aggregate":[14,90,210],"metrics":[15,24],"such":[16],"as":[17,58,126],"macro-F1":[18],"and":[19,65,81,104,162,186],"Cohen's":[20],"kappa.":[21],"However,":[22],"these":[23,194],"provide":[25],"limited":[26],"insight":[27],"into":[28],"how":[29,120],"performance":[31,116,211],"varies":[32],"across":[33],"student":[34],"responses":[35,139],"of":[36,61,74,165],"differing":[37],"difficulty.":[39,68,167],"We":[40,93],"introduce":[41],"an":[42],"evaluation":[43],"framework":[44,96,204],"for":[45,205],"LLM-based":[46,207],"ASAG":[47,208],"based":[48],"on":[49,101,137,142],"item":[50,198],"response":[51,66,127,199],"theory":[52,200],"(IRT),":[53],"which":[54],"correctness":[57],"a":[59,147,202],"function":[60],"latent":[62],"grader":[63],"ability":[64],"This":[69],"formulation":[70],"enables":[71],"response-level":[72],"analysis":[73],"where":[75],"LLM":[76],"graders":[77],"succeed":[78],"or":[79],"fail":[80],"reveals":[82],"robustness":[83],"differences":[84],"that":[85,110,135,197],"are":[86],"not":[87],"visible":[88],"from":[89],"scores":[91],"alone.":[92],"apply":[94],"the":[95,102,143,180],"to":[97,179],"17":[98],"open-weight":[99],"LLMs":[100],"SciEntsBank":[103],"Beetle":[105],"benchmarks.":[106],"The":[107],"results":[108,195],"show":[109,134,196],"even":[111],"similar":[114],"overall":[115],"differ":[117],"substantially":[118],"in":[119,190],"sharply":[121],"their":[122],"accuracy":[124],"declines":[125],"difficulty":[128,172],"increases.":[129],"In":[130],"addition,":[131],"confusion":[132],"patterns":[133],"errors":[136],"difficult":[138,156],"concentrate":[140],"disproportionately":[141],"\\texttt{partially\\_correct\\_incomplete}":[144],"label,":[145],"indicating":[146],"tendency":[148],"toward":[149],"intermediate-label":[150],"collapse":[151],"under":[152],"ambiguity.":[153],"To":[154],"characterize":[155],"responses,":[157],"we":[158],"further":[159],"analyze":[160],"semantic":[161,177,188],"linguistic":[163],"correlates":[164],"estimated":[166],"Across":[168],"both":[169],"datasets,":[170],"higher":[171],"associated":[174],"weaker":[176],"alignment":[178],"reference":[181],"answer,":[182],"stronger":[183],"contradiction":[184],"signals,":[185],"greater":[187],"isolation":[189],"embedding":[191],"space.":[192],"Overall,":[193],"offers":[201],"useful":[203],"evaluating":[206],"beyond":[209],"measures.":[212]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-05-05T00:00:00"}