{"id":"https://openalex.org/W7220928252","doi":"https://doi.org/10.48550/arxiv.2610.09460","title":"Beyond Score Accuracy: Examining the Diagnostic Quality of LLM-Generated Structured Assessment in Higher Education","display_name":"Beyond Score Accuracy: Examining the Diagnostic Quality of LLM-Generated Structured Assessment in Higher Education","publication_year":2026,"publication_date":"2026-10-07","ids":{"openalex":"https://openalex.org/W7220928252","doi":"https://doi.org/10.48550/arxiv.2610.09460"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2610.09460","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2610.09460","pdf_url":"https://arxiv.org/pdf/2610.09460","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2610.09460","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5157903621","display_name":"Xi Zhao","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I78577930","display_name":"Columbia University","ror":"https://ror.org/00hj8s172","country_code":"US","type":"education","lineage":["https://openalex.org/I78577930"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Zhao, Xi","raw_affiliation_strings":["Teachers College , Columbia University , USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Teachers College , Columbia University , USA","institution_ids":["https://openalex.org/I78577930"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5063704820","display_name":"Xinyue Jiao","orcid":"https://orcid.org/0000-0001-9496-609X","observed_orcids":["https://orcid.org/0000-0001-9496-609X"]},"institutions":[{"id":"https://openalex.org/I57206974","display_name":"New York University","ror":"https://ror.org/0190ak572","country_code":"US","type":"education","lineage":["https://openalex.org/I57206974"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jiao, Xinyue","raw_affiliation_strings":["School of Culture, Education, and Human Development , New York University , USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Culture, Education, and Human Development , New York University , USA","institution_ids":["https://openalex.org/I57206974"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5157897720","display_name":"Zhen Xu","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I78577930","display_name":"Columbia University","ror":"https://ror.org/00hj8s172","country_code":"US","type":"education","lineage":["https://openalex.org/I78577930"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Xu, Zhen","raw_affiliation_strings":["Teachers College , Columbia University , USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Teachers College , Columbia University , USA","institution_ids":["https://openalex.org/I78577930"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10959","display_name":"Student Assessment and Feedback","score":0.31619998812675476,"subfield":{"id":"https://openalex.org/subfields/3304","display_name":"Education"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10959","display_name":"Student Assessment and Feedback","score":0.31619998812675476,"subfield":{"id":"https://openalex.org/subfields/3304","display_name":"Education"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T14414","display_name":"Artificial Intelligence in Education","score":0.29760000109672546,"subfield":{"id":"https://openalex.org/subfields/3304","display_name":"Education"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11122","display_name":"Online Learning and Analytics","score":0.11680000275373459,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.9739999771118164},{"id":"https://openalex.org/keywords/automated-grading","display_name":"automated grading","score":0.9240000247955322},{"id":"https://openalex.org/keywords/llm-generated-feedback","display_name":"LLM-generated feedback","score":0.8320000171661377},{"id":"https://openalex.org/keywords/diagnostic-assessment","display_name":"diagnostic assessment","score":0.6050000190734863},{"id":"https://openalex.org/keywords/rubric-based-assessment","display_name":"rubric-based assessment","score":0.7480000257492065},{"id":"https://openalex.org/keywords/student-misconceptions","display_name":"student misconceptions","score":0.8190000057220459},{"id":"https://openalex.org/keywords/grading-accuracy","display_name":"grading accuracy","score":0.628000020980835},{"id":"https://openalex.org/keywords/computer-science-education","display_name":"computer science education","score":0.8809999823570251},{"id":"https://openalex.org/keywords/human-oversight","display_name":"human oversight","score":0.492000013589859},{"id":"https://openalex.org/keywords/higher-education-evaluation","display_name":"higher education assessment","score":0.45899999141693115}],"concepts":[{"id":"https://openalex.org/C111640148","wikidata":"https://www.wikidata.org/wiki/Q847349","display_name":"Rubric","level":2,"score":0.8955000042915344},{"id":"https://openalex.org/C2777286243","wikidata":"https://www.wikidata.org/wiki/Q5591926","display_name":"Grading (engineering)","level":2,"score":0.8676000237464905},{"id":"https://openalex.org/C2779356329","wikidata":"https://www.wikidata.org/wiki/Q922625","display_name":"Checklist","level":2,"score":0.6341999769210815},{"id":"https://openalex.org/C120912362","wikidata":"https://www.wikidata.org/wiki/Q136822","display_name":"Higher education","level":2,"score":0.6176000237464905},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.5486999750137329},{"id":"https://openalex.org/C509550671","wikidata":"https://www.wikidata.org/wiki/Q126945","display_name":"Medical education","level":1,"score":0.3930000066757202},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.3612000048160553},{"id":"https://openalex.org/C22156102","wikidata":"https://www.wikidata.org/wiki/Q5341294","display_name":"Educational measurement","level":3,"score":0.36000001430511475},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.31839999556541443},{"id":"https://openalex.org/C3018395757","wikidata":"https://www.wikidata.org/wiki/Q1379672","display_name":"Evaluation methods","level":2,"score":0.31040000915527344},{"id":"https://openalex.org/C75630572","wikidata":"https://www.wikidata.org/wiki/Q538904","display_name":"Applied psychology","level":1,"score":0.30649998784065247},{"id":"https://openalex.org/C145420912","wikidata":"https://www.wikidata.org/wiki/Q853077","display_name":"Mathematics education","level":1,"score":0.3025999963283539},{"id":"https://openalex.org/C61863361","wikidata":"https://www.wikidata.org/wiki/Q470749","display_name":"Inter-rater reliability","level":3,"score":0.29190000891685486},{"id":"https://openalex.org/C2781463719","wikidata":"https://www.wikidata.org/wiki/Q2002022","display_name":"Grade inflation","level":3,"score":0.28850001096725464},{"id":"https://openalex.org/C42525527","wikidata":"https://www.wikidata.org/wiki/Q1209955","display_name":"Formative assessment","level":2,"score":0.2791000008583069},{"id":"https://openalex.org/C188231857","wikidata":"https://www.wikidata.org/wiki/Q2585724","display_name":"Educational assessment","level":2,"score":0.2639000117778778}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2610.09460","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2610.09460","pdf_url":"https://arxiv.org/pdf/2610.09460","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2610.09460","doi":"https://doi.org/10.48550/arxiv.2610.09460","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2610.09460","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2610.09460","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2610.09460","pdf_url":"https://arxiv.org/pdf/2610.09460","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","display_name":"Quality education","score":0.8278999924659729}],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7220928252.pdf","grobid_xml":"https://content.openalex.org/works/W7220928252.grobid-xml"},"referenced_works_count":1,"referenced_works":["https://openalex.org/W4417414024"],"related_works":["https://openalex.org/W4416439933","https://openalex.org/W7161751804","https://openalex.org/W4405916612","https://openalex.org/W4412585228","https://openalex.org/W7204193965","https://openalex.org/W6963190205","https://openalex.org/W4406457822","https://openalex.org/W4415602515","https://openalex.org/W4408711855","https://openalex.org/W7140687447"],"abstract_inverted_index":{"As":[0],"Large":[1],"Language":[2],"Models":[3],"(LLMs)":[4],"are":[5,90],"increasingly":[6],"adopted":[7],"for":[8,117],"automated":[9],"grading":[10,84,148,171],"and":[11,26,67,79,85,172,178],"feedback":[12,24,109,130,173],"in":[13,144],"higher":[14],"education,":[15],"their":[16],"structured":[17],"outputs,":[18],"including":[19],"multi-dimensional":[20],"rubric":[21],"scores,":[22],"detailed":[23],"comments,":[25],"improvement":[27],"suggestions,":[28],"create":[29],"an":[30],"appearance":[31],"of":[32,52,82,136,169],"thorough":[33],"analytic":[34],"evaluation.":[35],"This":[36],"study":[37],"examines":[38],"whether":[39],"these":[40],"outputs":[41],"deliver":[42],"what":[43],"they":[44],"appear":[45],"to":[46,56,98],"offer.":[47],"Using":[48],"the":[49,77,140],"JorGPT":[50],"dataset":[51],"3,041":[53],"student":[54,112],"responses":[55],"50":[57],"open-ended":[58],"computer":[59],"science":[60],"questions,":[61],"scored":[62],"by":[63,152],"both":[64],"human":[65,145,182],"instructors":[66],"three":[68,73],"commercial":[69],"LLMs,":[70],"we":[71],"identify":[72],"systematic":[74],"discrepancies":[75],"between":[76],"apparent":[78],"actual":[80],"quality":[81],"LLM-generated":[83,170],"feedback.":[86,146],"The":[87,107,129],"sub-dimension":[88],"scores":[89],"highly":[91],"correlated":[92],"(r":[93],"=":[94],"0.82-0.99,":[95],"VIF":[96],"up":[97],"45),":[99],"providing":[100],"redundant":[101],"rather":[102,124],"than":[103,125],"independent":[104],"diagnostic":[105,127],"information.":[106],"textual":[108],"rarely":[110],"detects":[111],"misconceptions":[113],"(5-7%":[114],"vs.":[115],"15-31%":[116],"teachers),":[118],"functioning":[119],"as":[120],"a":[121,126],"coverage":[122],"checklist":[123],"instrument.":[128],"tone":[131],"remains":[132],"uniformly":[133],"positive":[134],"regardless":[135],"response":[137],"quality,":[138],"lacking":[139],"severity":[141],"modulation":[142],"observed":[143],"Additionally,":[147],"accuracy":[149],"varies":[150],"significantly":[151],"knowledge":[153],"domain,":[154],"with":[155],"procedural":[156],"topics":[157],"most":[158],"reliable.":[159],"These":[160],"findings":[161],"provide":[162],"empirically":[163],"grounded":[164],"guidance":[165],"on":[166],"which":[167,179],"aspects":[168],"can":[174],"be":[175],"relied":[176],"upon":[177],"require":[180],"continued":[181],"oversight.":[183]},"counts_by_year":[],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2026-10-09T00:00:00"}