{"id":"https://openalex.org/W7133818840","doi":"https://doi.org/10.22152/programming-journal.org/2026/11/5","title":"Evaluating LLMs in the Context of a Functional Programming Course: A Comprehensive Study","display_name":"Evaluating LLMs in the Context of a Functional Programming Course: A Comprehensive Study","publication_year":2026,"publication_date":"2026-02-15","ids":{"openalex":"https://openalex.org/W7133818840","doi":"https://doi.org/10.22152/programming-journal.org/2026/11/5"},"language":"en","primary_location":{"id":"doi:10.22152/programming-journal.org/2026/11/5","is_oa":false,"landing_page_url":"https://doi.org/10.22152/programming-journal.org/2026/11/5","pdf_url":null,"source":{"id":"https://openalex.org/S4210199781","display_name":"The Art Science and Engineering of Programming","issn_l":"2473-7321","issn":["2473-7321"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"The Art, Science, and Engineering of Programming","raw_type":"journal-article"},"type":"article","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2603.05646","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5124949573","display_name":"Yihan Zhang","orcid":null},"institutions":[{"id":"https://openalex.org/I5023651","display_name":"McGill University","ror":"https://ror.org/01pxwe438","country_code":"CA","type":"education","lineage":["https://openalex.org/I5023651"]}],"countries":["CA"],"is_corresponding":true,"raw_author_name":"Yihan Zhang","raw_affiliation_strings":["McGill University, Canada"],"affiliations":[{"raw_affiliation_string":"McGill University, Canada","institution_ids":["https://openalex.org/I5023651"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5044908798","display_name":"Brigitte Pientka","orcid":"https://orcid.org/0000-0002-2549-4276"},"institutions":[{"id":"https://openalex.org/I5023651","display_name":"McGill University","ror":"https://ror.org/01pxwe438","country_code":"CA","type":"education","lineage":["https://openalex.org/I5023651"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Brigitte Pientka","raw_affiliation_strings":["McGill University, Canada"],"affiliations":[{"raw_affiliation_string":"McGill University, Canada","institution_ids":["https://openalex.org/I5023651"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5128204958","display_name":"Xujie Si","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xujie Si","raw_affiliation_strings":["University of Toronto, USA"],"affiliations":[{"raw_affiliation_string":"University of Toronto, USA","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5124949573"],"corresponding_institution_ids":["https://openalex.org/I5023651"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.43362767,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"11","issue":"1","first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13034","display_name":"Information Systems Education and Curriculum Development","score":0.3018999993801117,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T13034","display_name":"Information Systems Education and Curriculum Development","score":0.3018999993801117,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10533","display_name":"Teaching and Learning Programming","score":0.051899999380111694,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12884","display_name":"Educational Assessment and Pedagogy","score":0.038100000470876694,"subfield":{"id":"https://openalex.org/subfields/3304","display_name":"Education"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.6251999735832214},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.24289999902248383}],"concepts":[{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.6251999735832214},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4503999948501587},{"id":"https://openalex.org/C56739046","wikidata":"https://www.wikidata.org/wiki/Q192060","display_name":"Knowledge management","level":1,"score":0.3995000123977661},{"id":"https://openalex.org/C112930515","wikidata":"https://www.wikidata.org/wiki/Q4389547","display_name":"Risk analysis (engineering)","level":1,"score":0.37450000643730164},{"id":"https://openalex.org/C539667460","wikidata":"https://www.wikidata.org/wiki/Q2414942","display_name":"Management science","level":1,"score":0.3643999993801117},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.35850000381469727},{"id":"https://openalex.org/C195094911","wikidata":"https://www.wikidata.org/wiki/Q14167904","display_name":"Process management","level":1,"score":0.3215000033378601},{"id":"https://openalex.org/C39549134","wikidata":"https://www.wikidata.org/wiki/Q133080","display_name":"Public relations","level":1,"score":0.2831999957561493},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.25949999690055847},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.24289999902248383}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.22152/programming-journal.org/2026/11/5","is_oa":false,"landing_page_url":"https://doi.org/10.22152/programming-journal.org/2026/11/5","pdf_url":null,"source":{"id":"https://openalex.org/S4210199781","display_name":"The Art Science and Engineering of Programming","issn_l":"2473-7321","issn":["2473-7321"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"The Art, Science, and Engineering of Programming","raw_type":"journal-article"},{"id":"pmh:oai:arXiv.org:2603.05646","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2603.05646","pdf_url":"https://arxiv.org/pdf/2603.05646","source":{"id":"https://openalex.org/S4393918464","display_name":"ArXiv.org","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:doi:10.48550/arxiv.2603.05646","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4406922384","display_name":"Open MIND","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"publisher-specific-oa","license_id":"https://openalex.org/licenses/publisher-specific-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2603.05646","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2603.05646","pdf_url":"https://arxiv.org/pdf/2603.05646","source":{"id":"https://openalex.org/S4393918464","display_name":"ArXiv.org","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Large-Language":[0],"Models":[1],"(LLMs)":[2],"are":[3,158],"changing":[4],"the":[5,11,47,52,125,145,148,154,177,236,252,265],"way":[6],"learners":[7],"acquire":[8],"knowledge":[9],"outside":[10],"classroom":[12],"setting.":[13,64],"Previous":[14],"studies":[15],"have":[16],"shown":[17],"that":[18,153],"LLMs":[19,50,118,157,195,213,283],"seem":[20],"effective":[21,159],"in":[22,29,51,61,176,188,197,221],"generating":[23,206],"to":[24,71,122,143,181,208,250,257,271,278],"short":[25],"and":[26,95,128,190,200,270,284,290],"simple":[27],"questions":[28,110],"introductory":[30,185],"CS":[31],"courses":[32],"using":[33,124,140],"high-resource":[34],"programming":[35,57,84,113,167,186,232,273],"languages":[36,233],"such":[37],"as":[38,203,205,226],"Java":[39],"or":[40],"Python.":[41],"In":[42,65],"this":[43],"paper,":[44],"we":[45,67],"evaluate":[46,73],"effectiveness":[48],"of":[49,54,147,194,238,255,267],"context":[53],"a":[55,164,227],"low-resource":[56,178,294],"language":[58,109,274],"--":[59],"OCaml,":[60],"an":[62,129],"educational":[63],"particular,":[66],"built":[68],"three":[69,156],"benchmarks":[70,245],"comprehensively":[72],"9":[74],"state-of-the-art":[75],"LLMs:":[76],"1)":[77],"$\u03bb$CodeGen":[78],"(a":[79,88,105],"benchmark":[80,89,106],"containing":[81,90,107],"natural-language":[82],"homework":[83,174],"problems);":[85],"2)":[86],"$\u03bb$Repair":[87],"programs":[91],"with":[92,120],"syntax,":[93],"type,":[94],"logical":[96],"errors":[97,202],"drawn":[98],"from":[99],"actual":[100],"student":[101],"submissions);":[102],"3)":[103],"$\u03bb$Explain":[104],"natural":[108],"regarding":[111],"theoretical":[112],"concepts).":[114],"We":[115,242],"grade":[116],"each":[117],"responses":[119],"respect":[121],"correctness":[123],"OCaml":[126],"compiler":[127],"autograder.":[130],"And":[131],"our":[132,244],"evaluation":[133,137],"goes":[134],"beyond":[135],"common":[136],"methodology":[138],"by":[139],"manual":[141],"grading":[142],"assess":[144,251],"quality":[146],"responses.":[149],"Our":[150],"study":[151],"shows":[152],"top":[155],"on":[160,184],"all":[161],"tasks":[162],"within":[163],"typical":[165],"functional":[166],"course,":[168],"although":[169],"they":[170],"solve":[171],"much":[172],"fewer":[173],"problems":[175,187],"setting":[179],"compared":[180],"their":[182,224],"success":[183],"Python":[189],"Java.":[191],"The":[192],"strength":[193],"lies":[196],"correcting":[198],"syntax":[199],"type":[201],"well":[204],"answers":[207],"basic":[209],"conceptual":[210],"questions.":[211],"While":[212],"may":[214],"not":[215],"yet":[216],"match":[217],"dedicated":[218],"language-specific":[219],"tools":[220,292],"some":[222],"areas,":[223],"convenience":[225],"one-stop":[228],"tool":[229],"for":[230,293],"multiple":[231,248],"can":[234,246],"outweigh":[235],"benefits":[237],"more":[239,286],"specialized":[240],"systems.":[241],"hope":[243],"serve":[247],"purposes:":[249],"evolving":[253],"capabilities":[254],"LLMs,":[256],"help":[258],"instructors":[259],"raise":[260],"awareness":[261],"among":[262],"students":[263],"about":[264,276],"limitations":[266],"LLM-generated":[268],"solutions,":[269],"inform":[272],"researchers":[275],"opportunities":[277],"integrate":[279],"domain-specific":[280],"reasoning":[281],"into":[282],"develop":[285],"powerful":[287],"code":[288],"synthesis":[289],"repair":[291],"languages.":[295]},"counts_by_year":[],"updated_date":"2026-04-04T16:13:02.066488","created_date":"2026-03-06T00:00:00"}