{"id":"https://openalex.org/W4412781285","doi":"https://doi.org/10.1145/3702652.3744220","title":"Rubric Is All You Need: Improving LLM-Based Code Evaluation With Question-Specific Rubrics","display_name":"Rubric Is All You Need: Improving LLM-Based Code Evaluation With Question-Specific Rubrics","publication_year":2025,"publication_date":"2025-07-31","ids":{"openalex":"https://openalex.org/W4412781285","doi":"https://doi.org/10.1145/3702652.3744220"},"language":"en","primary_location":{"id":"doi:10.1145/3702652.3744220","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3702652.3744220","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2025 ACM Conference on International Computing Education Research V.1","raw_type":"proceedings-article"},"type":"preprint","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3702652.3744220","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Aditya Pathak","orcid":"https://orcid.org/0009-0003-4712-1790"},"institutions":[{"id":"https://openalex.org/I74796645","display_name":"Birla Institute of Technology and Science, Pilani","ror":"https://ror.org/001p3jz28","country_code":"IN","type":"education","lineage":["https://openalex.org/I74796645"]}],"countries":["IN"],"is_corresponding":true,"raw_author_name":"Aditya Pathak","raw_affiliation_strings":["BITS Pilani, Pilani, India"],"raw_orcid":"https://orcid.org/0009-0003-4712-1790","affiliations":[{"raw_affiliation_string":"BITS Pilani, Pilani, India","institution_ids":["https://openalex.org/I74796645"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Rachit Gandhi","orcid":"https://orcid.org/0009-0006-1594-6052"},"institutions":[{"id":"https://openalex.org/I74796645","display_name":"Birla Institute of Technology and Science, Pilani","ror":"https://ror.org/001p3jz28","country_code":"IN","type":"education","lineage":["https://openalex.org/I74796645"]}],"countries":["IN"],"is_corresponding":false,"raw_author_name":"Rachit Gandhi","raw_affiliation_strings":["BITS Pilani, Pilani, India"],"raw_orcid":"https://orcid.org/0009-0006-1594-6052","affiliations":[{"raw_affiliation_string":"BITS Pilani, Pilani, India","institution_ids":["https://openalex.org/I74796645"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5028526395","display_name":"V. Gowthami Uttam","orcid":null},"institutions":[{"id":"https://openalex.org/I74796645","display_name":"Birla Institute of Technology and Science, Pilani","ror":"https://ror.org/001p3jz28","country_code":"IN","type":"education","lineage":["https://openalex.org/I74796645"]}],"countries":["IN"],"is_corresponding":false,"raw_author_name":"Vaibhav Uttam","raw_affiliation_strings":["BITS Pilani, Pilani, India"],"raw_orcid":"https://orcid.org/0009-0002-6660-6374","affiliations":[{"raw_affiliation_string":"BITS Pilani, Pilani, India","institution_ids":["https://openalex.org/I74796645"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5119133566","display_name":"Arnav Ramamoorthy","orcid":null},"institutions":[{"id":"https://openalex.org/I74796645","display_name":"Birla Institute of Technology and Science, Pilani","ror":"https://ror.org/001p3jz28","country_code":"IN","type":"education","lineage":["https://openalex.org/I74796645"]}],"countries":["IN"],"is_corresponding":false,"raw_author_name":"Arnav Ramamoorthy","raw_affiliation_strings":["BITS Pilani, Pilani, India"],"raw_orcid":"https://orcid.org/0009-0003-5577-9138","affiliations":[{"raw_affiliation_string":"BITS Pilani, Pilani, India","institution_ids":["https://openalex.org/I74796645"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Pratyush Ghosh","orcid":"https://orcid.org/0009-0002-2313-2949"},"institutions":[{"id":"https://openalex.org/I74796645","display_name":"Birla Institute of Technology and Science, Pilani","ror":"https://ror.org/001p3jz28","country_code":"IN","type":"education","lineage":["https://openalex.org/I74796645"]}],"countries":["IN"],"is_corresponding":false,"raw_author_name":"Pratyush Ghosh","raw_affiliation_strings":["BITS Pilani, Pilani, India"],"raw_orcid":"https://orcid.org/0009-0002-2313-2949","affiliations":[{"raw_affiliation_string":"BITS Pilani, Pilani, India","institution_ids":["https://openalex.org/I74796645"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Aaryan Raj Jindal","orcid":"https://orcid.org/0009-0001-7509-9389"},"institutions":[{"id":"https://openalex.org/I74796645","display_name":"Birla Institute of Technology and Science, Pilani","ror":"https://ror.org/001p3jz28","country_code":"IN","type":"education","lineage":["https://openalex.org/I74796645"]}],"countries":["IN"],"is_corresponding":false,"raw_author_name":"Aaryan Raj Jindal","raw_affiliation_strings":["BITS Pilani, Pilani, India"],"raw_orcid":"https://orcid.org/0009-0001-7509-9389","affiliations":[{"raw_affiliation_string":"BITS Pilani, Pilani, India","institution_ids":["https://openalex.org/I74796645"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Shreyash Verma","orcid":"https://orcid.org/0009-0008-7239-2986"},"institutions":[{"id":"https://openalex.org/I74796645","display_name":"Birla Institute of Technology and Science, Pilani","ror":"https://ror.org/001p3jz28","country_code":"IN","type":"education","lineage":["https://openalex.org/I74796645"]}],"countries":["IN"],"is_corresponding":false,"raw_author_name":"Shreyash Verma","raw_affiliation_strings":["BITS Pilani, Pilani, India"],"raw_orcid":"https://orcid.org/0009-0008-7239-2986","affiliations":[{"raw_affiliation_string":"BITS Pilani, Pilani, India","institution_ids":["https://openalex.org/I74796645"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Aditya Mittal","orcid":"https://orcid.org/0009-0000-6497-7142"},"institutions":[{"id":"https://openalex.org/I74796645","display_name":"Birla Institute of Technology and Science, Pilani","ror":"https://ror.org/001p3jz28","country_code":"IN","type":"education","lineage":["https://openalex.org/I74796645"]}],"countries":["IN"],"is_corresponding":false,"raw_author_name":"Aditya Mittal","raw_affiliation_strings":["BITS Pilani, Pilani, India"],"raw_orcid":"https://orcid.org/0009-0000-6497-7142","affiliations":[{"raw_affiliation_string":"BITS Pilani, Pilani, India","institution_ids":["https://openalex.org/I74796645"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5119133567","display_name":"Aashna Ased","orcid":null},"institutions":[{"id":"https://openalex.org/I74796645","display_name":"Birla Institute of Technology and Science, Pilani","ror":"https://ror.org/001p3jz28","country_code":"IN","type":"education","lineage":["https://openalex.org/I74796645"]}],"countries":["IN"],"is_corresponding":false,"raw_author_name":"Aashna Ased","raw_affiliation_strings":["BITS Pilani, Pilani, India"],"raw_orcid":"https://orcid.org/0009-0002-1027-3873","affiliations":[{"raw_affiliation_string":"BITS Pilani, Pilani, India","institution_ids":["https://openalex.org/I74796645"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5040014991","display_name":"Chirag Khatri","orcid":null},"institutions":[{"id":"https://openalex.org/I74796645","display_name":"Birla Institute of Technology and Science, Pilani","ror":"https://ror.org/001p3jz28","country_code":"IN","type":"education","lineage":["https://openalex.org/I74796645"]}],"countries":["IN"],"is_corresponding":false,"raw_author_name":"Chirag Khatri","raw_affiliation_strings":["BITS Pilani, Pilani, India"],"raw_orcid":"https://orcid.org/0009-0004-8346-4966","affiliations":[{"raw_affiliation_string":"BITS Pilani, Pilani, India","institution_ids":["https://openalex.org/I74796645"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5119133564","display_name":"Yashwanth Nakka","orcid":null},"institutions":[{"id":"https://openalex.org/I74796645","display_name":"Birla Institute of Technology and Science, Pilani","ror":"https://ror.org/001p3jz28","country_code":"IN","type":"education","lineage":["https://openalex.org/I74796645"]}],"countries":["IN"],"is_corresponding":false,"raw_author_name":"Yashwanth Nakka","raw_affiliation_strings":["BITS Pilani, Pilani, India"],"raw_orcid":"https://orcid.org/0009-0007-1509-8644","affiliations":[{"raw_affiliation_string":"BITS Pilani, Pilani, India","institution_ids":["https://openalex.org/I74796645"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5119133565","display_name":"Devansh","orcid":null},"institutions":[{"id":"https://openalex.org/I74796645","display_name":"Birla Institute of Technology and Science, Pilani","ror":"https://ror.org/001p3jz28","country_code":"IN","type":"education","lineage":["https://openalex.org/I74796645"]}],"countries":["IN"],"is_corresponding":false,"raw_author_name":"Devansh","raw_affiliation_strings":["BITS Pilani, Pilani, India"],"raw_orcid":"https://orcid.org/0009-0006-6882-2379","affiliations":[{"raw_affiliation_string":"BITS Pilani, Pilani, India","institution_ids":["https://openalex.org/I74796645"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5044760764","display_name":"Jagat Sesh Challa","orcid":"https://orcid.org/0000-0002-9794-0087"},"institutions":[{"id":"https://openalex.org/I74796645","display_name":"Birla Institute of Technology and Science, Pilani","ror":"https://ror.org/001p3jz28","country_code":"IN","type":"education","lineage":["https://openalex.org/I74796645"]}],"countries":["IN"],"is_corresponding":false,"raw_author_name":"Jagat Sesh Challa","raw_affiliation_strings":["BITS Pilani, Pilani, India"],"raw_orcid":"https://orcid.org/0000-0002-9794-0087","affiliations":[{"raw_affiliation_string":"BITS Pilani, Pilani, India","institution_ids":["https://openalex.org/I74796645"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100780534","display_name":"Dhruv Kumar","orcid":"https://orcid.org/0000-0003-4831-1847"},"institutions":[{"id":"https://openalex.org/I74796645","display_name":"Birla Institute of Technology and Science, Pilani","ror":"https://ror.org/001p3jz28","country_code":"IN","type":"education","lineage":["https://openalex.org/I74796645"]}],"countries":["IN"],"is_corresponding":false,"raw_author_name":"Dhruv Kumar","raw_affiliation_strings":["BITS Pilani, Pilani, India"],"raw_orcid":"https://orcid.org/0000-0003-4831-1847","affiliations":[{"raw_affiliation_string":"BITS Pilani, Pilani, India","institution_ids":["https://openalex.org/I74796645"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":14,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I74796645"],"apc_list":null,"apc_paid":null,"fwci":11.9558,"has_fulltext":false,"cited_by_count":8,"citation_normalized_percentile":{"value":0.98332138,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"181","last_page":"195"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10533","display_name":"Teaching and Learning Programming","score":0.9972000122070312,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10533","display_name":"Teaching and Learning Programming","score":0.9972000122070312,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9897000193595886,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12171","display_name":"Open Education and E-Learning","score":0.9178000092506409,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/rubric","display_name":"Rubric","score":0.9839229583740234},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7860690951347351},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.5225169658660889},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.36641037464141846},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.33618080615997314},{"id":"https://openalex.org/keywords/mathematics-education","display_name":"Mathematics education","score":0.17532426118850708},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.09984377026557922}],"concepts":[{"id":"https://openalex.org/C111640148","wikidata":"https://www.wikidata.org/wiki/Q847349","display_name":"Rubric","level":2,"score":0.9839229583740234},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7860690951347351},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.5225169658660889},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.36641037464141846},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.33618080615997314},{"id":"https://openalex.org/C145420912","wikidata":"https://www.wikidata.org/wiki/Q853077","display_name":"Mathematics education","level":1,"score":0.17532426118850708},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.09984377026557922},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3702652.3744220","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3702652.3744220","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2025 ACM Conference on International Computing Education Research V.1","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2503.23989","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2503.23989","pdf_url":"https://arxiv.org/pdf/2503.23989","source":{"id":"https://openalex.org/S4393918464","display_name":"ArXiv.org","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"}],"best_oa_location":{"id":"doi:10.1145/3702652.3744220","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3702652.3744220","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2025 ACM Conference on International Computing Education Research V.1","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G2740752811","display_name":null,"funder_award_id":"N4/24/1004","funder_id":"https://openalex.org/F4320323889","funder_display_name":"Birla Institute of Technology and Science, Pilani"}],"funders":[{"id":"https://openalex.org/F4320323889","display_name":"Birla Institute of Technology and Science, Pilani","ror":"https://ror.org/001p3jz28"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":31,"referenced_works":["https://openalex.org/W1544206805","https://openalex.org/W1607283167","https://openalex.org/W2101105183","https://openalex.org/W2151828850","https://openalex.org/W2254299877","https://openalex.org/W2293412016","https://openalex.org/W2465021265","https://openalex.org/W2747427841","https://openalex.org/W2894540915","https://openalex.org/W2906729407","https://openalex.org/W2954129953","https://openalex.org/W2972161276","https://openalex.org/W3011338620","https://openalex.org/W4225108562","https://openalex.org/W4283705032","https://openalex.org/W4380353816","https://openalex.org/W4381586920","https://openalex.org/W4386617189","https://openalex.org/W4386719008","https://openalex.org/W4389524484","https://openalex.org/W4390621504","https://openalex.org/W4392564705","https://openalex.org/W4392617381","https://openalex.org/W4392934670","https://openalex.org/W4400267714","https://openalex.org/W4403474856","https://openalex.org/W4405821763","https://openalex.org/W4406327959","https://openalex.org/W4406459916","https://openalex.org/W4406619802","https://openalex.org/W4407681796"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W1987813225","https://openalex.org/W4390610055","https://openalex.org/W4379053243","https://openalex.org/W3021020626","https://openalex.org/W2955201676","https://openalex.org/W2153005530","https://openalex.org/W3199392376"],"abstract_inverted_index":{"Since":[0],"the":[1,10,56,69,81,90],"emergence":[2],"of":[3,12,34,92,172],"Large":[4],"Language":[5],"Models":[6],"(LLMs)":[7],"popularized":[8],"by":[9],"release":[11],"GPT-3":[13],"and":[14,51,103,115,119],"ChatGPT,":[15],"LLMs":[16,28,39],"have":[17],"shown":[18],"remarkable":[19],"promise":[20],"in":[21,55,174],"programming-related":[22],"tasks.":[23],"While":[24],"code":[25,36,49,173],"generation":[26],"using":[27,38,64,137],"has":[29],"become":[30],"a":[31,100,111,147],"popular":[32,112],"field":[33],"research,":[35],"evaluation":[37,50,94,155],"remains":[40],"under-explored.":[41],"In":[42,134],"this":[43],"paper,":[44],"we":[45,96,144],"focus":[46],"on":[47],"LLM-based":[48],"attempt":[52],"to":[53,68,136,158],"fill":[54],"existing":[57,82],"gaps.":[58],"We":[59],"propose":[60,146],"multi-agentic":[61],"novel":[62],"approaches":[63,83],"\\emph{question-specific":[65,166],"rubrics}":[66,167],"tailored":[67],"problem":[70],"statement,":[71],"arguing":[72],"that":[73,84,165],"these":[74],"perform":[75],"better":[76,178],"for":[77],"logical":[78,170],"assessment":[79,171],"than":[80],"use":[85],"\\emph{question-agnostic":[86],"rubrics}.":[87],"To":[88],"address":[89],"lack":[91],"suitable":[93],"datasets,":[95],"introduce":[97],"two":[98],"datasets:":[99],"Data":[101,113],"Structures":[102,114],"Algorithms":[104,116],"dataset":[105,124],"containing":[106],"150":[107],"student":[108,127],"submissions":[109,128],"from":[110,129],"practice":[117],"website,":[118],"an":[120],"Object":[121],"Oriented":[122],"Programming":[123],"comprising":[125],"80":[126],"undergraduate":[130],"computer":[131],"science":[132],"courses.":[133],"addition":[135],"standard":[138],"metrics":[139],"(Spearman":[140],"Correlation,":[141],"Cohen's":[142],"Kappa),":[143],"additionally":[145],"new":[148],"metric":[149],"called":[150],"as":[151],"Leniency,":[152],"which":[153],"quantifies":[154],"strictness":[156],"relative":[157],"expert":[159],"assessment.":[160],"Our":[161],"comprehensive":[162],"analysis":[163],"demonstrates":[164],"significantly":[168],"enhance":[169],"educational":[175],"settings,":[176],"providing":[177],"feedback":[179],"aligned":[180],"with":[181],"instructional":[182],"goals":[183],"beyond":[184],"mere":[185],"syntactic":[186],"correctness.":[187]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":5}],"updated_date":"2026-04-25T08:17:42.794288","created_date":"2025-07-31T00:00:00"}