{"id":"https://openalex.org/W7103993747","doi":"https://doi.org/10.1109/tse.2025.3629828","title":"Literate Programming With LLMs? \u2014 A Study on Rosetta Code and CodeNet","display_name":"Literate Programming With LLMs? \u2014 A Study on Rosetta Code and CodeNet","publication_year":2025,"publication_date":"2025-11-05","ids":{"openalex":"https://openalex.org/W7103993747","doi":"https://doi.org/10.1109/tse.2025.3629828"},"language":null,"primary_location":{"id":"doi:10.1109/tse.2025.3629828","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tse.2025.3629828","pdf_url":null,"source":{"id":"https://openalex.org/S8351582","display_name":"IEEE Transactions on Software Engineering","issn_l":"0098-5589","issn":["0098-5589","1939-3520","2326-3881"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Software Engineering","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Simin Sun","orcid":"https://orcid.org/0009-0003-0873-5968"},"institutions":[{"id":"https://openalex.org/I66862912","display_name":"Chalmers University of Technology","ror":"https://ror.org/040wg7k59","country_code":"SE","type":"education","lineage":["https://openalex.org/I66862912"]},{"id":"https://openalex.org/I881427289","display_name":"University of Gothenburg","ror":"https://ror.org/01tm6cn81","country_code":"SE","type":"education","lineage":["https://openalex.org/I881427289"]}],"countries":["SE"],"is_corresponding":false,"raw_author_name":"Simin Sun","raw_affiliation_strings":["Department of Computer Science and Engineering, Chalmers University of Technology and University of Gothenburg, Gothenburg, Sweden"],"raw_orcid":"https://orcid.org/0009-0003-0873-5968","affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering, Chalmers University of Technology and University of Gothenburg, Gothenburg, Sweden","institution_ids":["https://openalex.org/I66862912","https://openalex.org/I881427289"]}]},{"author_position":"last","author":{"id":null,"display_name":"Miroslaw Staron","orcid":"https://orcid.org/0000-0002-9052-0864"},"institutions":[{"id":"https://openalex.org/I66862912","display_name":"Chalmers University of Technology","ror":"https://ror.org/040wg7k59","country_code":"SE","type":"education","lineage":["https://openalex.org/I66862912"]},{"id":"https://openalex.org/I881427289","display_name":"University of Gothenburg","ror":"https://ror.org/01tm6cn81","country_code":"SE","type":"education","lineage":["https://openalex.org/I881427289"]}],"countries":["SE"],"is_corresponding":false,"raw_author_name":"Miroslaw Staron","raw_affiliation_strings":["Department of Computer Science and Engineering, Chalmers University of Technology and University of Gothenburg, Gothenburg, Sweden"],"raw_orcid":"https://orcid.org/0000-0002-9052-0864","affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering, Chalmers University of Technology and University of Gothenburg, Gothenburg, Sweden","institution_ids":["https://openalex.org/I66862912","https://openalex.org/I881427289"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.61635985,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"52","issue":"2","first_page":"468","last_page":"487"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.6430000066757202,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.6430000066757202,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10533","display_name":"Teaching and Learning Programming","score":0.07209999859333038,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12377","display_name":"Digital Humanities and Scholarship","score":0.024900000542402267,"subfield":{"id":"https://openalex.org/subfields/1208","display_name":"Literature and Literary Theory"},"field":{"id":"https://openalex.org/fields/12","display_name":"Arts and Humanities"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/semantics","display_name":"Semantics (computer science)","score":0.5817999839782715},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.4973999857902527},{"id":"https://openalex.org/keywords/programming-paradigm","display_name":"Programming paradigm","score":0.46860000491142273},{"id":"https://openalex.org/keywords/representation","display_name":"Representation (politics)","score":0.4235000014305115},{"id":"https://openalex.org/keywords/class","display_name":"Class (philosophy)","score":0.40549999475479126},{"id":"https://openalex.org/keywords/encode","display_name":"ENCODE","score":0.3935999870300293},{"id":"https://openalex.org/keywords/documentation","display_name":"Documentation","score":0.3637000024318695},{"id":"https://openalex.org/keywords/fifth-generation-programming-language","display_name":"Fifth-generation programming language","score":0.36329999566078186},{"id":"https://openalex.org/keywords/second-generation-programming-language","display_name":"Second-generation programming language","score":0.3621000051498413}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8414000272750854},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.8205000162124634},{"id":"https://openalex.org/C184337299","wikidata":"https://www.wikidata.org/wiki/Q1437428","display_name":"Semantics (computer science)","level":2,"score":0.5817999839782715},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.4973999857902527},{"id":"https://openalex.org/C34165917","wikidata":"https://www.wikidata.org/wiki/Q188267","display_name":"Programming paradigm","level":2,"score":0.46860000491142273},{"id":"https://openalex.org/C2776359362","wikidata":"https://www.wikidata.org/wiki/Q2145286","display_name":"Representation (politics)","level":3,"score":0.4235000014305115},{"id":"https://openalex.org/C2777212361","wikidata":"https://www.wikidata.org/wiki/Q5127848","display_name":"Class (philosophy)","level":2,"score":0.40549999475479126},{"id":"https://openalex.org/C66746571","wikidata":"https://www.wikidata.org/wiki/Q1134833","display_name":"ENCODE","level":3,"score":0.3935999870300293},{"id":"https://openalex.org/C56666940","wikidata":"https://www.wikidata.org/wiki/Q788790","display_name":"Documentation","level":2,"score":0.3637000024318695},{"id":"https://openalex.org/C199305712","wikidata":"https://www.wikidata.org/wiki/Q3205914","display_name":"Fifth-generation programming language","level":3,"score":0.36329999566078186},{"id":"https://openalex.org/C11164408","wikidata":"https://www.wikidata.org/wiki/Q18657800","display_name":"Second-generation programming language","level":4,"score":0.3621000051498413},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.36059999465942383},{"id":"https://openalex.org/C33857546","wikidata":"https://www.wikidata.org/wiki/Q80006","display_name":"Computer programming","level":2,"score":0.35659998655319214},{"id":"https://openalex.org/C529173508","wikidata":"https://www.wikidata.org/wiki/Q638608","display_name":"Software development","level":3,"score":0.3449999988079071},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.33980000019073486},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.33980000019073486},{"id":"https://openalex.org/C50033165","wikidata":"https://www.wikidata.org/wiki/Q15712089","display_name":"Inductive programming","level":3,"score":0.3334999978542328},{"id":"https://openalex.org/C133162039","wikidata":"https://www.wikidata.org/wiki/Q1061077","display_name":"Code generation","level":3,"score":0.33059999346733093},{"id":"https://openalex.org/C74149592","wikidata":"https://www.wikidata.org/wiki/Q762446","display_name":"Comparison of multi-paradigm programming languages","level":5,"score":0.3301999866962433},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.3116999864578247},{"id":"https://openalex.org/C39890363","wikidata":"https://www.wikidata.org/wiki/Q36108","display_name":"Generative grammar","level":2,"score":0.31060001254081726},{"id":"https://openalex.org/C18701968","wikidata":"https://www.wikidata.org/wiki/Q2670534","display_name":"Programming language theory","level":4,"score":0.2879999876022339},{"id":"https://openalex.org/C206146517","wikidata":"https://www.wikidata.org/wiki/Q1294375","display_name":"Third-generation programming language","level":5,"score":0.27790001034736633},{"id":"https://openalex.org/C169590947","wikidata":"https://www.wikidata.org/wiki/Q47506","display_name":"Compiler","level":2,"score":0.275299996137619},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.26829999685287476},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.26489999890327454},{"id":"https://openalex.org/C77660490","wikidata":"https://www.wikidata.org/wiki/Q244916","display_name":"Intermediate language","level":3,"score":0.2619999945163727}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tse.2025.3629828","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tse.2025.3629828","pdf_url":null,"source":{"id":"https://openalex.org/S8351582","display_name":"IEEE Transactions on Software Engineering","issn_l":"0098-5589","issn":["0098-5589","1939-3520","2326-3881"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Software Engineering","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.8993769288063049,"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4"}],"awards":[{"id":"https://openalex.org/G2855260886","display_name":null,"funder_award_id":"2025-22-467","funder_id":"https://openalex.org/F4320322581","funder_display_name":"Vetenskapsr\u00e5det"},{"id":"https://openalex.org/G4169454217","display_name":null,"funder_award_id":"2024-04687","funder_id":"https://openalex.org/F4320322581","funder_display_name":"Vetenskapsr\u00e5det"},{"id":"https://openalex.org/G4624230512","display_name":null,"funder_award_id":"2025-22-1115","funder_id":"https://openalex.org/F4320322581","funder_display_name":"Vetenskapsr\u00e5det"},{"id":"https://openalex.org/G7390188278","display_name":null,"funder_award_id":"2025-22-1040","funder_id":"https://openalex.org/F4320322581","funder_display_name":"Vetenskapsr\u00e5det"},{"id":"https://openalex.org/G8823848175","display_name":null,"funder_award_id":"2024-22-1517","funder_id":"https://openalex.org/F4320322581","funder_display_name":"Vetenskapsr\u00e5det"}],"funders":[{"id":"https://openalex.org/F4320322581","display_name":"Vetenskapsr\u00e5det","ror":"https://ror.org/03zttf063"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":33,"referenced_works":["https://openalex.org/W1846534677","https://openalex.org/W1870798554","https://openalex.org/W1879090356","https://openalex.org/W2012401665","https://openalex.org/W2087634802","https://openalex.org/W2124807415","https://openalex.org/W2264869955","https://openalex.org/W2344444819","https://openalex.org/W2765872439","https://openalex.org/W2796040126","https://openalex.org/W2946359678","https://openalex.org/W2964150020","https://openalex.org/W2964204621","https://openalex.org/W2964268484","https://openalex.org/W2970641574","https://openalex.org/W3034689979","https://openalex.org/W3091730360","https://openalex.org/W3098605233","https://openalex.org/W3108032709","https://openalex.org/W3174885139","https://openalex.org/W3184324824","https://openalex.org/W3202070718","https://openalex.org/W4205596491","https://openalex.org/W4214548153","https://openalex.org/W4242395378","https://openalex.org/W4291213652","https://openalex.org/W4385562549","https://openalex.org/W4385572925","https://openalex.org/W4389606799","https://openalex.org/W4392414327","https://openalex.org/W4393106737","https://openalex.org/W4402516191","https://openalex.org/W4413267893"],"related_works":[],"abstract_inverted_index":{"Literate":[0],"programming,":[1,67,163],"a":[2,24,108,190],"concept":[3],"introduced":[4],"by":[5],"Knuth":[6],"in":[7,60,155,166],"1984,":[8],"emphasized":[9],"the":[10,111,123,139,156,167,198,208],"importance":[11],"of":[12,52,56,89,161,200,211],"combining":[13],"human-readable":[14],"documentation":[15],"with":[16,31,77],"machine-readable":[17],"code":[18,76],"as":[19],"writing":[20],"literate":[21,53,66,162,212],"programs":[22,90],"is":[23,34],"prerequisite":[25],"for":[26],"software":[27,61],"quality.":[28],"Our":[29,150],"objective":[30],"this":[32],"paper":[33],"to":[35,94,193,206],"evaluate":[36,103],"whether":[37],"generative":[38],"AI":[39],"models,":[40],"Large":[41],"Language":[42],"Models":[43],"(LLM)":[44],"like":[45],"GPT-4,":[46],"LLaMA":[47],"or":[48],"Falcon,":[49],"are":[50,159,172],"capable":[51,160],"programming":[54,97,132,176,195,201],"because":[55],"their":[57,86,100,104],"extensive":[58],"use":[59],"engineering.":[62],"To":[63,102],"truly":[64],"achieve":[65],"LLMs":[68,154,188],"must":[69],"generate":[70],"natural":[71],"language":[72],"descriptions":[73],"and":[74,99,114,134,147,169,197],"corresponding":[75],"aligned":[78],"semantics":[79,199],"based":[80],"on":[81,138,181],"user":[82],"prompts.":[83],"In":[84],"addition,":[85],"internal":[87],"representation":[88],"should":[91],"allow":[92],"us":[93,204],"recognize":[95],"both":[96],"languages":[98,177,196],"descriptions.":[101],"capabilities,":[105],"we":[106,184],"conducted":[107],"study":[109],"using":[110,122],"Rosetta":[112,124],"Code":[113,125],"CodeNet":[115,141],"repositories.":[116],"We":[117],"perform":[118],"four":[119],"computational":[120],"experiments":[121],"repository,":[126],"encompassing":[127],"1,228":[128],"tasks":[129,146],"across":[130],"926":[131],"languages,":[133],"validate":[135],"our":[136],"findings":[137,151],"larger":[140],"dataset,":[142],"which":[143],"includes":[144],"55":[145],"52":[148],"languages.":[149],"show":[152],"that":[153,186],"trillion-parameter":[157],"class":[158],"while":[164],"models":[165],"million-":[168],"billion-parameter":[170],"classes":[171],"better":[173],"at":[174],"recognizing":[175],"than":[178],"tasks.":[179],"Based":[180],"these":[182],"results,":[183],"conclude":[185],"modern":[187],"inhibit":[189],"deeper":[191],"ability":[192],"encode":[194],"tasks,":[202],"bringing":[203],"closer":[205],"realizing":[207],"full":[209],"potential":[210],"programming.":[213]},"counts_by_year":[],"updated_date":"2026-02-13T13:36:01.753593","created_date":"2025-11-06T00:00:00"}