{"id":"https://openalex.org/W7106716581","doi":"https://doi.org/10.48550/arxiv.2511.18597","title":"Toward Trustworthy Difficulty Assessments: Large Language Models as Judges in Programming and Synthetic Tasks","display_name":"Toward Trustworthy Difficulty Assessments: Large Language Models as Judges in Programming and Synthetic Tasks","publication_year":2025,"publication_date":"2025-11-23","ids":{"openalex":"https://openalex.org/W7106716581","doi":"https://doi.org/10.48550/arxiv.2511.18597"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2511.18597","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2511.18597","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2511.18597","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Tabib, H. M. Shadman","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tabib, H. M. Shadman","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":null,"display_name":"Deedar, Jaber Ahmed","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Deedar, Jaber Ahmed","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.19539999961853027,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},"topics":[{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.19539999961853027,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T13629","display_name":"Text Readability and Simplification","score":0.1518000066280365,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.14990000426769257,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/downgrade","display_name":"Downgrade","score":0.6525999903678894},{"id":"https://openalex.org/keywords/trustworthiness","display_name":"Trustworthiness","score":0.5856999754905701},{"id":"https://openalex.org/keywords/natural-language","display_name":"Natural language","score":0.492900013923645},{"id":"https://openalex.org/keywords/natural-language-understanding","display_name":"Natural language understanding","score":0.4569000005722046},{"id":"https://openalex.org/keywords/synthetic-data","display_name":"Synthetic data","score":0.37860000133514404},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.37630000710487366},{"id":"https://openalex.org/keywords/confusion","display_name":"Confusion","score":0.3528999984264374}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.762499988079071},{"id":"https://openalex.org/C2779628075","wikidata":"https://www.wikidata.org/wiki/Q1253258","display_name":"Downgrade","level":2,"score":0.6525999903678894},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6007999777793884},{"id":"https://openalex.org/C153701036","wikidata":"https://www.wikidata.org/wiki/Q659974","display_name":"Trustworthiness","level":2,"score":0.5856999754905701},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.492900013923645},{"id":"https://openalex.org/C2779439875","wikidata":"https://www.wikidata.org/wiki/Q1078276","display_name":"Natural language understanding","level":3,"score":0.4569000005722046},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.44519999623298645},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3937999904155731},{"id":"https://openalex.org/C160920958","wikidata":"https://www.wikidata.org/wiki/Q7662746","display_name":"Synthetic data","level":2,"score":0.37860000133514404},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.37630000710487366},{"id":"https://openalex.org/C2781140086","wikidata":"https://www.wikidata.org/wiki/Q557945","display_name":"Confusion","level":2,"score":0.3528999984264374},{"id":"https://openalex.org/C34165917","wikidata":"https://www.wikidata.org/wiki/Q188267","display_name":"Programming paradigm","level":2,"score":0.3472999930381775},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.3418999910354614},{"id":"https://openalex.org/C26760741","wikidata":"https://www.wikidata.org/wiki/Q160402","display_name":"Perception","level":2,"score":0.30329999327659607},{"id":"https://openalex.org/C2776401178","wikidata":"https://www.wikidata.org/wiki/Q12050496","display_name":"Feature (linguistics)","level":2,"score":0.2964000105857849},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.2800999879837036},{"id":"https://openalex.org/C15077461","wikidata":"https://www.wikidata.org/wiki/Q7248507","display_name":"Programming language specification","level":5,"score":0.27730000019073486},{"id":"https://openalex.org/C24858836","wikidata":"https://www.wikidata.org/wiki/Q844718","display_name":"Theory of computation","level":2,"score":0.2676999866962433},{"id":"https://openalex.org/C2776608160","wikidata":"https://www.wikidata.org/wiki/Q4785462","display_name":"Natural (archaeology)","level":2,"score":0.26350000500679016}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2511.18597","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2511.18597","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2511.18597","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2511.18597","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article"},"sustainable_development_goals":[{"score":0.8691421747207642,"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Large":[0],"Language":[1],"Models":[2],"(LLMs)":[3],"have":[4],"demonstrated":[5],"impressive":[6],"capabilities":[7],"in":[8,117,184,204],"natural":[9],"language":[10],"and":[11,14,24,67,96,109,131,180,186,188],"code":[12],"generation,":[13],"are":[15],"increasingly":[16],"deployed":[17],"as":[18,34,53,105,160],"automatic":[19,181],"judges":[20,199],"of":[21,38,49,73,154],"model":[22],"outputs":[23],"learning":[25],"activities.":[26],"Yet,":[27],"their":[28],"behavior":[29],"on":[30,64,178],"structured":[31],"tasks":[32],"such":[33,104],"predicting":[35],"the":[36],"difficulty":[37,56,182],"competitive":[39,205],"programming":[40,185],"problems":[41,76,120,159,169],"remains":[42],"under-explored.":[43],"We":[44,139],"conduct":[45],"a":[46,54,71,114,133,144],"systematic":[47],"comparison":[48],"GPT-4o,":[50],"used":[51],"purely":[52],"natural-language":[55],"assessor,":[57],"against":[58],"an":[59],"interpretable":[60],"Light-GBM":[61],"ensemble":[62],"trained":[63],"explicit":[65],"numeric":[66,101],"textual":[68],"features.":[69],"On":[70],"dataset":[72],"1,825":[74],"LeetCode":[75],"labeled":[77],"Easy,":[78],"Medium,":[79,161],"or":[80,209],"Hard,":[81],"LightGBM":[82],"attains":[83],"86%":[84],"accuracy,":[85],"whereas":[86],"GPT-4o":[87,126,142,150],"reaches":[88],"only":[89],"37.75%.":[90],"Detailed":[91],"analyses,":[92],"including":[93],"confusion":[94],"matrices":[95],"SHAP-based":[97],"interpretability,":[98],"show":[99],"that":[100,193],"constraints":[102],"--":[103,112],"input":[106],"size":[107],"limits":[108],"acceptance":[110],"rates":[111],"play":[113],"crucial":[115],"role":[116],"separating":[118],"Hard":[119,158,168],"from":[121],"easier":[122],"ones.":[123],"By":[124],"contrast,":[125],"often":[127],"overlooks":[128],"these":[129],"cues":[130],"exhibits":[132],"strong":[134],"bias":[135],"toward":[136],"simpler":[137],"categories.":[138],"further":[140],"probe":[141],"through":[143],"synthetic":[145,157],"Hard-problem":[146],"generation":[147],"protocol.":[148],"Surprisingly,":[149],"labels":[151],"almost":[152],"all":[153],"its":[155,163],"own":[156],"contradicting":[162],"tendency":[164],"to":[165,170,175],"downgrade":[166],"real":[167],"Easy.":[171],"Our":[172],"findings":[173],"connect":[174],"recent":[176],"work":[177],"LLMs-as-judges":[179],"estimation":[183],"education,":[187],"highlight":[189],"concrete":[190],"failure":[191],"modes":[192],"must":[194],"be":[195,201],"addressed":[196],"before":[197],"LLM-based":[198],"can":[200],"considered":[202],"trustworthy":[203],"programming,":[206],"educational":[207],"platforms,":[208],"reinforcement-learning":[210],"pipelines.":[211]},"counts_by_year":[],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-11-27T00:00:00"}