{"id":"https://openalex.org/W7130604683","doi":"https://doi.org/10.48550/arxiv.2602.16039","title":"How Uncertain Is the Grade? A Benchmark of Uncertainty Metrics for LLM-Based Automatic Assessment","display_name":"How Uncertain Is the Grade? A Benchmark of Uncertainty Metrics for LLM-Based Automatic Assessment","publication_year":2026,"publication_date":"2026-02-17","ids":{"openalex":"https://openalex.org/W7130604683","doi":"https://doi.org/10.48550/arxiv.2602.16039"},"language":null,"primary_location":{"id":"pmh:doi:10.48550/arxiv.2602.16039","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4406922384","display_name":"Open MIND","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"publisher-specific-oa","license_id":"https://openalex.org/licenses/publisher-specific-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":null,"any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5126389623","display_name":"Hang Li","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Hang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5126379756","display_name":"Kaiqi Yang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yang, Kaiqi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5114988070","display_name":"Xianxuan Long","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Long, Xianxuan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5126376098","display_name":"Fedor Filippov","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Filippov, Fedor","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5126438417","display_name":"Yucheng Chu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chu, Yucheng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5114452226","display_name":"Yasemin Copur-Gencturk","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Copur-Gencturk, Yasemin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5126387228","display_name":"Peng He","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"He, Peng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5126443024","display_name":"Cory Miller","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Miller, Cory","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5126431650","display_name":"Namsoo Shin","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shin, Namsoo","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5126445162","display_name":"Joseph Krajcik","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Krajcik, Joseph","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5126418380","display_name":"Hui Liu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Hui","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5126408745","display_name":"Jiliang Tang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tang, Jiliang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11902","display_name":"Intelligent Tutoring Systems and Adaptive Learning","score":0.2526000142097473,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11902","display_name":"Intelligent Tutoring Systems and Adaptive Learning","score":0.2526000142097473,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.15729999542236328,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10959","display_name":"Student Assessment and Feedback","score":0.06520000100135803,"subfield":{"id":"https://openalex.org/subfields/3304","display_name":"Education"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/probabilistic-logic","display_name":"Probabilistic logic","score":0.6804999709129333},{"id":"https://openalex.org/keywords/adaptability","display_name":"Adaptability","score":0.5748000144958496},{"id":"https://openalex.org/keywords/flexibility","display_name":"Flexibility (engineering)","score":0.5715000033378601},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.5250999927520752},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.4643000066280365},{"id":"https://openalex.org/keywords/grading","display_name":"Grading (engineering)","score":0.4472000002861023},{"id":"https://openalex.org/keywords/uncertainty-quantification","display_name":"Uncertainty quantification","score":0.43070000410079956},{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.41519999504089355}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7156000137329102},{"id":"https://openalex.org/C49937458","wikidata":"https://www.wikidata.org/wiki/Q2599292","display_name":"Probabilistic logic","level":2,"score":0.6804999709129333},{"id":"https://openalex.org/C177606310","wikidata":"https://www.wikidata.org/wiki/Q5674297","display_name":"Adaptability","level":2,"score":0.5748000144958496},{"id":"https://openalex.org/C2780598303","wikidata":"https://www.wikidata.org/wiki/Q65921492","display_name":"Flexibility (engineering)","level":2,"score":0.5715000033378601},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.5250999927520752},{"id":"https://openalex.org/C112930515","wikidata":"https://www.wikidata.org/wiki/Q4389547","display_name":"Risk analysis (engineering)","level":1,"score":0.4787999987602234},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.4643000066280365},{"id":"https://openalex.org/C2777286243","wikidata":"https://www.wikidata.org/wiki/Q5591926","display_name":"Grading (engineering)","level":2,"score":0.4472000002861023},{"id":"https://openalex.org/C32230216","wikidata":"https://www.wikidata.org/wiki/Q7882499","display_name":"Uncertainty quantification","level":2,"score":0.43070000410079956},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4242999851703644},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.41519999504089355},{"id":"https://openalex.org/C2776889888","wikidata":"https://www.wikidata.org/wiki/Q1135789","display_name":"Unintended consequences","level":2,"score":0.41429999470710754},{"id":"https://openalex.org/C12174686","wikidata":"https://www.wikidata.org/wiki/Q1058438","display_name":"Risk assessment","level":2,"score":0.38370001316070557},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.3813000023365021},{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.3483000099658966},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.34630000591278076},{"id":"https://openalex.org/C177803969","wikidata":"https://www.wikidata.org/wiki/Q29205","display_name":"Uncertainty analysis","level":2,"score":0.2985000014305115},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.29159998893737793},{"id":"https://openalex.org/C204323151","wikidata":"https://www.wikidata.org/wiki/Q905424","display_name":"Range (aeronautics)","level":2,"score":0.2824000120162964},{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.28130000829696655},{"id":"https://openalex.org/C137209882","wikidata":"https://www.wikidata.org/wiki/Q1403517","display_name":"Measurement uncertainty","level":2,"score":0.2630000114440918},{"id":"https://openalex.org/C2775924081","wikidata":"https://www.wikidata.org/wiki/Q55608371","display_name":"Control (management)","level":2,"score":0.25200000405311584}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:doi:10.48550/arxiv.2602.16039","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4406922384","display_name":"Open MIND","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"publisher-specific-oa","license_id":"https://openalex.org/licenses/publisher-specific-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"},{"id":"doi:10.48550/arxiv.2602.16039","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2602.16039","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:doi:10.48550/arxiv.2602.16039","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4406922384","display_name":"Open MIND","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"publisher-specific-oa","license_id":"https://openalex.org/licenses/publisher-specific-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education","score":0.8233747482299805}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"The":[0],"rapid":[1],"rise":[2],"of":[3,12,49,120,127,134,162,196,204,226],"large":[4],"language":[5],"models":[6],"(LLMs)":[7],"is":[8,53],"reshaping":[9],"the":[10,45,125,132,177,192,202,224,234,246],"landscape":[11],"automatic":[13,58,129,155,230],"assessment":[14,61,167,210,231],"in":[15,23,31,57,68,102,124,140,150,183,228,245],"education.":[16],"While":[17],"these":[18,135,188],"systems":[19,244],"demonstrate":[20],"substantial":[21],"advantages":[22],"adaptability":[24],"to":[25,40,77,91],"diverse":[26],"question":[27],"types":[28],"and":[29,100,111,148,171,194,200,212,232,240],"flexibility":[30],"output":[32,41],"formats,":[33],"they":[34],"also":[35],"introduce":[36],"new":[37],"challenges":[38],"related":[39],"uncertainty,":[42],"stemming":[43],"from":[44],"inherently":[46],"probabilistic":[47],"nature":[48],"LLMs.":[50],"Output":[51],"uncertainty":[52,87,121,163,178,198,216,227],"an":[54],"inescapable":[55],"challenge":[56,110],"assessment,":[59],"as":[60,74],"results":[62],"often":[63],"play":[64],"a":[65,117],"critical":[66],"role":[67],"informing":[69],"subsequent":[70],"pedagogical":[71],"actions,":[72],"such":[73],"providing":[75],"feedback":[76],"students":[78],"or":[79,84],"guiding":[80],"instructional":[81],"decisions.":[82],"Unreliable":[83],"poorly":[85],"calibrated":[86],"estimates":[88],"can":[89],"lead":[90],"unstable":[92],"downstream":[93],"interventions,":[94],"potentially":[95],"disrupting":[96],"students'":[97],"learning":[98],"processes":[99],"resulting":[101],"unintended":[103],"negative":[104],"consequences.":[105],"To":[106],"systematically":[107],"understand":[108],"this":[109],"inform":[112],"future":[113],"research,":[114],"we":[115,175,190],"benchmark":[116],"broad":[118],"range":[119],"quantification":[122],"methods":[123,136],"context":[126],"LLM-based":[128,229],"assessment.":[130],"Although":[131],"effectiveness":[133],"has":[137],"been":[138],"demonstrated":[139],"many":[141],"tasks":[142],"across":[143,165],"other":[144],"domains,":[145],"their":[146],"applicability":[147],"reliability":[149],"educational":[151],"settings,":[152,174],"particularly":[153],"for":[154,236],"grading,":[156],"remain":[157],"underexplored.":[158],"Through":[159],"comprehensive":[160],"analyses":[161],"behaviors":[164],"multiple":[166],"datasets,":[168],"LLM":[169],"families,":[170,209],"generation":[172],"control":[173],"characterize":[176],"patterns":[179],"exhibited":[180],"by":[181],"LLMs":[182],"grading":[184,243],"scenarios.":[185],"Based":[186],"on":[187,215],"findings,":[189],"evaluate":[191],"strengths":[193],"limitations":[195],"different":[197],"metrics":[199],"analyze":[201],"influence":[203],"key":[205],"factors,":[206],"including":[207],"model":[208],"tasks,":[211],"decoding":[213],"strategies,":[214],"estimates.":[217],"Our":[218],"study":[219],"provides":[220],"actionable":[221],"insights":[222],"into":[223],"characteristics":[225],"lays":[233],"groundwork":[235],"developing":[237],"more":[238],"reliable":[239],"effective":[241],"uncertainty-aware":[242],"future.":[247]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-02-20T00:00:00"}