{"id":"https://openalex.org/W7167643773","doi":"https://doi.org/10.1145/3786181.3788709","title":"LLM-Powered On-Demand Test Suites in Self-Graded Student Programming Assignments","display_name":"LLM-Powered On-Demand Test Suites in Self-Graded Student Programming Assignments","publication_year":2026,"publication_date":"2026-04-12","ids":{"openalex":"https://openalex.org/W7167643773","doi":"https://doi.org/10.1145/3786181.3788709"},"language":null,"primary_location":{"id":"doi:10.1145/3786181.3788709","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3786181.3788709","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 3rd International Workshop on Large Language Models For Code","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3786181.3788709","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100353280","display_name":"Chang Liu","orcid":"https://orcid.org/0000-0002-6721-1959"},"institutions":[{"id":"https://openalex.org/I4210106879","display_name":"Ohio University","ror":"https://ror.org/01jr3y717","country_code":"US","type":"education","lineage":["https://openalex.org/I4210106879"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Chang Liu","raw_affiliation_strings":["Ohio University, Athens, Ohio, USA"],"raw_orcid":"https://orcid.org/0000-0002-6721-1959","affiliations":[{"raw_affiliation_string":"Ohio University, Athens, Ohio, USA","institution_ids":["https://openalex.org/I4210106879"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5100353280"],"corresponding_institution_ids":["https://openalex.org/I4210106879"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.9388432,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"192","last_page":"196"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10533","display_name":"Teaching and Learning Programming","score":0.8823000192642212,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10533","display_name":"Teaching and Learning Programming","score":0.8823000192642212,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.053599998354911804,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10430","display_name":"Software Engineering Techniques and Practices","score":0.010900000110268593,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/test-suite","display_name":"Test suite","score":0.7544999718666077},{"id":"https://openalex.org/keywords/test","display_name":"Test (biology)","score":0.5831000208854675},{"id":"https://openalex.org/keywords/dilemma","display_name":"Dilemma","score":0.5080000162124634},{"id":"https://openalex.org/keywords/suite","display_name":"Suite","score":0.49880000948905945},{"id":"https://openalex.org/keywords/class","display_name":"Class (philosophy)","score":0.4702000021934509},{"id":"https://openalex.org/keywords/automatic-programming","display_name":"Automatic programming","score":0.4207000136375427},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.4203999936580658},{"id":"https://openalex.org/keywords/pair-programming","display_name":"Pair programming","score":0.3977999985218048}],"concepts":[{"id":"https://openalex.org/C151552104","wikidata":"https://www.wikidata.org/wiki/Q7705809","display_name":"Test suite","level":4,"score":0.7544999718666077},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7293999791145325},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.6680999994277954},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.5831000208854675},{"id":"https://openalex.org/C2778496695","wikidata":"https://www.wikidata.org/wiki/Q254128","display_name":"Dilemma","level":2,"score":0.5080000162124634},{"id":"https://openalex.org/C79581498","wikidata":"https://www.wikidata.org/wiki/Q1367530","display_name":"Suite","level":2,"score":0.49880000948905945},{"id":"https://openalex.org/C2777212361","wikidata":"https://www.wikidata.org/wiki/Q5127848","display_name":"Class (philosophy)","level":2,"score":0.4702000021934509},{"id":"https://openalex.org/C2777394884","wikidata":"https://www.wikidata.org/wiki/Q762268","display_name":"Automatic programming","level":2,"score":0.4207000136375427},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.4203999936580658},{"id":"https://openalex.org/C174063052","wikidata":"https://www.wikidata.org/wiki/Q607013","display_name":"Pair programming","level":4,"score":0.3977999985218048},{"id":"https://openalex.org/C128942645","wikidata":"https://www.wikidata.org/wiki/Q1568346","display_name":"Test case","level":3,"score":0.3774000108242035},{"id":"https://openalex.org/C148027188","wikidata":"https://www.wikidata.org/wiki/Q907375","display_name":"Unit testing","level":3,"score":0.3280999958515167},{"id":"https://openalex.org/C120558287","wikidata":"https://www.wikidata.org/wiki/Q1418502","display_name":"Procedural programming","level":4,"score":0.30550000071525574},{"id":"https://openalex.org/C34165917","wikidata":"https://www.wikidata.org/wiki/Q188267","display_name":"Programming paradigm","level":2,"score":0.29490000009536743},{"id":"https://openalex.org/C77781355","wikidata":"https://www.wikidata.org/wiki/Q223335","display_name":"Structured programming","level":2,"score":0.29179999232292175},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.2883000075817108},{"id":"https://openalex.org/C2779760435","wikidata":"https://www.wikidata.org/wiki/Q5396169","display_name":"Minor (academic)","level":2,"score":0.28290000557899475},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.2759999930858612},{"id":"https://openalex.org/C50033165","wikidata":"https://www.wikidata.org/wiki/Q15712089","display_name":"Inductive programming","level":3,"score":0.2754000127315521},{"id":"https://openalex.org/C119263510","wikidata":"https://www.wikidata.org/wiki/Q7248501","display_name":"Programming domain","level":4,"score":0.27250000834465027},{"id":"https://openalex.org/C4478048","wikidata":"https://www.wikidata.org/wiki/Q950250","display_name":"Test-driven development","level":4,"score":0.2606000006198883},{"id":"https://openalex.org/C173404611","wikidata":"https://www.wikidata.org/wiki/Q528588","display_name":"Constraint programming","level":3,"score":0.251800000667572}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3786181.3788709","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3786181.3788709","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 3rd International Workshop on Large Language Models For Code","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3786181.3788709","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3786181.3788709","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 3rd International Workshop on Large Language Models For Code","raw_type":"proceedings-article"},"sustainable_development_goals":[{"score":0.7448965907096863,"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":9,"referenced_works":["https://openalex.org/W4385572142","https://openalex.org/W4391579642","https://openalex.org/W4391974543","https://openalex.org/W4392414327","https://openalex.org/W4402457546","https://openalex.org/W4411113098","https://openalex.org/W4411450140","https://openalex.org/W7124939946","https://openalex.org/W7133198585"],"related_works":[],"abstract_inverted_index":{"In":[0],"self-graded":[1],"student":[2,115],"programming":[3,80,116],"assignments,":[4],"it":[5],"is":[6,121],"a":[7,103],"dilemma":[8],"whether":[9],"to":[10,34,50,61,78,126],"include":[11,51],"randomly":[12],"generated":[13],"test":[14,57,68,95,109],"inputs":[15],"and":[16,55],"code":[17],"that":[18,71,119],"can":[19],"generate":[20],"correct":[21],"outputs":[22],"based":[23],"on":[24],"inputs.":[25],"Not":[26],"including":[27,40],"them":[28,41],"makes":[29],"the":[30,44],"tests":[31],"too":[32],"easy":[33],"bypass":[35],"or":[36],"be":[37,127],"tampered":[38],"with;":[39],"essentially":[42],"reveals":[43],"solution.":[45],"We":[46,75],"propose":[47],"an":[48,52,83],"approach":[49],"LLM-powered,":[53],"on-demand,":[54],"automated":[56],"suite":[58,69],"generation":[59,70],"mechanism":[60],"support":[62],"not":[63],"completely":[64],"random":[65],"but":[66],"hard-to-predict":[67],"provides":[72],"reasonable":[73],"coverage.":[74],"applied":[76,128],"this":[77,124],"seven":[79],"assignments":[81,117],"in":[82,93,108,114,131],"\"Organization":[84],"of":[85],"Programming":[86],"Languages\"":[87],"class":[88],"with":[89],"100%":[90],"success":[91,113],"rate":[92],"LLM-based":[94],"case":[96],"design,":[97],"although":[98],"we":[99],"did":[100],"run":[101],"into":[102],"few":[104],"minor":[105],"issues":[106],"occasionally":[107],"execution.":[110],"Our":[111],"initial":[112],"shows":[118],"there":[120],"potential":[122],"for":[123],"technique":[125],"more":[129],"broadly":[130],"test-driven":[132],"development":[133],"processes.":[134]},"counts_by_year":[],"updated_date":"2026-07-08T06:22:20.291053","created_date":"2026-07-08T00:00:00"}