{"id":"https://openalex.org/W4399577158","doi":"https://doi.org/10.1145/3650105.3652289","title":"Investigating the Performance of Language Models for Completing Code in Functional Programming Languages: a Haskell Case Study","display_name":"Investigating the Performance of Language Models for Completing Code in Functional Programming Languages: a Haskell Case Study","publication_year":2024,"publication_date":"2024-04-14","ids":{"openalex":"https://openalex.org/W4399577158","doi":"https://doi.org/10.1145/3650105.3652289"},"language":"en","primary_location":{"id":"doi:10.1145/3650105.3652289","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3650105.3652289","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3650105.3652289","source":null,"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2024 IEEE/ACM First International Conference on AI Foundation Models and Software Engineering","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3650105.3652289","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5074259780","display_name":"Tim van Dam","orcid":"https://orcid.org/0009-0009-3659-7068"},"institutions":[{"id":"https://openalex.org/I98358874","display_name":"Delft University of Technology","ror":"https://ror.org/02e2c7k09","country_code":"NL","type":"education","lineage":["https://openalex.org/I98358874"]}],"countries":["NL"],"is_corresponding":true,"raw_author_name":"Tim van Dam","raw_affiliation_strings":["Delft University of Technology, Delft, Netherlands"],"raw_orcid":"https://orcid.org/0009-0009-3659-7068","affiliations":[{"raw_affiliation_string":"Delft University of Technology, Delft, Netherlands","institution_ids":["https://openalex.org/I98358874"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5071240358","display_name":"Frank van der Heijden","orcid":"https://orcid.org/0009-0006-5922-1221"},"institutions":[{"id":"https://openalex.org/I98358874","display_name":"Delft University of Technology","ror":"https://ror.org/02e2c7k09","country_code":"NL","type":"education","lineage":["https://openalex.org/I98358874"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Frank van der Heijden","raw_affiliation_strings":["Delft University of Technology, Delft, Netherlands"],"raw_orcid":"https://orcid.org/0009-0006-5922-1221","affiliations":[{"raw_affiliation_string":"Delft University of Technology, Delft, Netherlands","institution_ids":["https://openalex.org/I98358874"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5053983805","display_name":"Philippe de Bekker","orcid":"https://orcid.org/0000-0002-3722-5428"},"institutions":[{"id":"https://openalex.org/I98358874","display_name":"Delft University of Technology","ror":"https://ror.org/02e2c7k09","country_code":"NL","type":"education","lineage":["https://openalex.org/I98358874"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Philippe de Bekker","raw_affiliation_strings":["Delft University of Technology, Delft, Netherlands"],"raw_orcid":"https://orcid.org/0000-0002-3722-5428","affiliations":[{"raw_affiliation_string":"Delft University of Technology, Delft, Netherlands","institution_ids":["https://openalex.org/I98358874"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5094246436","display_name":"Berend Nieuwschepen","orcid":"https://orcid.org/0009-0008-8146-550X"},"institutions":[{"id":"https://openalex.org/I98358874","display_name":"Delft University of Technology","ror":"https://ror.org/02e2c7k09","country_code":"NL","type":"education","lineage":["https://openalex.org/I98358874"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Berend Nieuwschepen","raw_affiliation_strings":["Delft University of Technology, Delft, Netherlands"],"raw_orcid":"https://orcid.org/0009-0008-8146-550X","affiliations":[{"raw_affiliation_string":"Delft University of Technology, Delft, Netherlands","institution_ids":["https://openalex.org/I98358874"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5094018467","display_name":"Marc Otten","orcid":null},"institutions":[{"id":"https://openalex.org/I98358874","display_name":"Delft University of Technology","ror":"https://ror.org/02e2c7k09","country_code":"NL","type":"education","lineage":["https://openalex.org/I98358874"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Marc Otten","raw_affiliation_strings":["Delft University of Technology, Delft, Netherlands"],"raw_orcid":"https://orcid.org/0009-0006-6525-8127","affiliations":[{"raw_affiliation_string":"Delft University of Technology, Delft, Netherlands","institution_ids":["https://openalex.org/I98358874"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5064355563","display_name":"Maliheh Izadi","orcid":"https://orcid.org/0000-0001-5093-5523"},"institutions":[{"id":"https://openalex.org/I98358874","display_name":"Delft University of Technology","ror":"https://ror.org/02e2c7k09","country_code":"NL","type":"education","lineage":["https://openalex.org/I98358874"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Maliheh Izadi","raw_affiliation_strings":["Delft University of Technology, Delft, Netherlands"],"raw_orcid":"https://orcid.org/0000-0001-5093-5523","affiliations":[{"raw_affiliation_string":"Delft University of Technology, Delft, Netherlands","institution_ids":["https://openalex.org/I98358874"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5074259780"],"corresponding_institution_ids":["https://openalex.org/I98358874"],"apc_list":null,"apc_paid":null,"fwci":4.476,"has_fulltext":false,"cited_by_count":6,"citation_normalized_percentile":{"value":0.9477735,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"91","last_page":"102"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.984000027179718,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9793000221252441,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/haskell","display_name":"Haskell","score":0.9907032251358032},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8704773783683777},{"id":"https://openalex.org/keywords/functional-programming","display_name":"Functional programming","score":0.8136264085769653},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.8034002780914307},{"id":"https://openalex.org/keywords/python","display_name":"Python (programming language)","score":0.6739789843559265},{"id":"https://openalex.org/keywords/javascript","display_name":"JavaScript","score":0.5776522159576416},{"id":"https://openalex.org/keywords/programming-paradigm","display_name":"Programming paradigm","score":0.4416431486606598}],"concepts":[{"id":"https://openalex.org/C2780624054","wikidata":"https://www.wikidata.org/wiki/Q34010","display_name":"Haskell","level":3,"score":0.9907032251358032},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8704773783683777},{"id":"https://openalex.org/C42383842","wikidata":"https://www.wikidata.org/wiki/Q193076","display_name":"Functional programming","level":2,"score":0.8136264085769653},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.8034002780914307},{"id":"https://openalex.org/C519991488","wikidata":"https://www.wikidata.org/wiki/Q28865","display_name":"Python (programming language)","level":2,"score":0.6739789843559265},{"id":"https://openalex.org/C544833334","wikidata":"https://www.wikidata.org/wiki/Q2005","display_name":"JavaScript","level":2,"score":0.5776522159576416},{"id":"https://openalex.org/C34165917","wikidata":"https://www.wikidata.org/wiki/Q188267","display_name":"Programming paradigm","level":2,"score":0.4416431486606598}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3650105.3652289","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3650105.3652289","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3650105.3652289","source":null,"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2024 IEEE/ACM First International Conference on AI Foundation Models and Software Engineering","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3650105.3652289","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3650105.3652289","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3650105.3652289","source":null,"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2024 IEEE/ACM First International Conference on AI Foundation Models and Software Engineering","raw_type":"proceedings-article"},"sustainable_development_goals":[{"score":0.699999988079071,"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4399577158.pdf"},"referenced_works_count":19,"referenced_works":["https://openalex.org/W2911468826","https://openalex.org/W2963341956","https://openalex.org/W2979792666","https://openalex.org/W2981852735","https://openalex.org/W2988647680","https://openalex.org/W3022049116","https://openalex.org/W3098605233","https://openalex.org/W3108032709","https://openalex.org/W3138429261","https://openalex.org/W3174220201","https://openalex.org/W3211801722","https://openalex.org/W4205596491","https://openalex.org/W4220672926","https://openalex.org/W4221166942","https://openalex.org/W4225108562","https://openalex.org/W4288089799","https://openalex.org/W4312438588","https://openalex.org/W4367672983","https://openalex.org/W4384026730"],"related_works":["https://openalex.org/W1461715167","https://openalex.org/W2112263414","https://openalex.org/W1602185604","https://openalex.org/W2028779182","https://openalex.org/W4242678340","https://openalex.org/W2140654465","https://openalex.org/W2129253059","https://openalex.org/W1488646450","https://openalex.org/W4296274916","https://openalex.org/W3004782279"],"abstract_inverted_index":{"Language":[0],"model-based":[1],"code":[2,15,24,140,191],"completion":[3,25,141],"models":[4,26,50,74,91,109,188],"have":[5],"quickly":[6],"grown":[7],"in":[8,16,39,126],"use,":[9],"helping":[10],"thousands":[11],"of":[12,42,71,122,129],"developers":[13],"write":[14],"many":[17],"different":[18],"programming":[19,46,83,124],"languages.":[20,47],"However,":[21],"research":[22],"on":[23,29,54,80,92,102,142,160,197],"typically":[27],"focuses":[28],"imperative":[30,123],"languages":[31,56,125,144],"such":[32,57],"as":[33,58],"Python":[34],"and":[35,78,88,168,189],"JavaScript,":[36],"which":[37],"results":[38],"a":[40,97],"lack":[41],"representation":[43],"for":[44,75,152],"functional":[45,55,82,136,143],"Consequently,":[48,147],"these":[49],"often":[51,174],"perform":[52],"poorly":[53],"Haskell.":[59,85],"To":[60],"investigate":[61],"whether":[62],"this":[63,148],"can":[64],"be":[65],"alleviated,":[66],"we":[67,105,181],"evaluate":[68,89,107],"the":[69,81,90,108,127,150,186],"performance":[70],"two":[72],"language":[73,84],"code,":[76],"CodeGPT":[77,163],"UniXcoder,":[79],"We":[86],"fine-tune":[87],"Haskell":[93,100,155],"functions":[94],"sourced":[95],"from":[96],"publicly":[98],"accessible":[99],"dataset":[101],"HuggingFace.":[103],"Additionally,":[104],"manually":[106],"using":[110],"our":[111,195],"novel":[112],"translated":[113],"HumanEval":[114],"dataset.":[115],"Our":[116],"automatic":[117],"evaluation":[118,159],"shows":[119,149],"that":[120,139],"knowledge":[121],"pre-training":[128],"LLMs":[130],"may":[131],"not":[132],"transfer":[133],"well":[134],"to":[135,193],"languages,":[137],"but":[138],"is":[145],"feasible.":[146],"need":[151],"more":[153,173],"high-quality":[154],"datasets.":[156],"A":[157],"manual":[158],"HumanEval-Haskell":[161],"indicates":[162],"frequently":[164],"generates":[165],"empty":[166],"predictions":[167],"extra":[169],"comments,":[170],"while":[171],"UniXcoder":[172],"produces":[175],"incomplete":[176],"or":[177],"incorrect":[178],"predictions.":[179],"Finally,":[180],"release":[182],"HumanEval-Haskell,":[183],"along":[184],"with":[185],"fine-tuned":[187],"all":[190],"required":[192],"reproduce":[194],"experiments":[196],"GitHub":[198],"[41].":[199]},"counts_by_year":[{"year":2025,"cited_by_count":5},{"year":2024,"cited_by_count":1}],"updated_date":"2026-03-27T05:58:40.876381","created_date":"2025-10-10T00:00:00"}