{"id":"https://openalex.org/W4411271504","doi":"https://doi.org/10.1109/msr66628.2025.00082","title":"Prompt Engineering or Fine-Tuning: An Empirical Assessment of LLMs for Code","display_name":"Prompt Engineering or Fine-Tuning: An Empirical Assessment of LLMs for Code","publication_year":2025,"publication_date":"2025-04-28","ids":{"openalex":"https://openalex.org/W4411271504","doi":"https://doi.org/10.1109/msr66628.2025.00082"},"language":"en","primary_location":{"id":"doi:10.1109/msr66628.2025.00082","doi":"https://doi.org/10.1109/msr66628.2025.00082","is_oa":false,"landing_page_url":"https://doi.org/10.1109/msr66628.2025.00082","pdf_url":null,"source":{"id":"https://openalex.org/S7407088599","display_name":"International Conference on Mining Software Repositories (MSR)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 IEEE/ACM 22nd International Conference on Mining Software Repositories (MSR)","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","is_traditional":true,"indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2310.10508","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5102446495","display_name":"Jiho Shin","orcid":"https://orcid.org/0000-0001-8829-3773","observed_orcids":["https://orcid.org/0000-0001-8829-3773"]},"institutions":[{"id":"https://openalex.org/I192455969","display_name":"York University","ror":"https://ror.org/05fq50484","country_code":"CA","type":"education","lineage":["https://openalex.org/I192455969"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Jiho Shin","raw_affiliation_strings":["York University,Lassonde School of Engineering,Toronto,Canada"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"York University,Lassonde School of Engineering,Toronto,Canada","institution_ids":["https://openalex.org/I192455969"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102568509","display_name":"C.J. Tang","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Clark Tang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5093081461","display_name":"Tahmineh Mohati","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I168635309","display_name":"University of Calgary","ror":"https://ror.org/03yjb2x39","country_code":"CA","type":"education","lineage":["https://openalex.org/I168635309"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Tahmineh Mohati","raw_affiliation_strings":["University of Calgary,Schulich School of Engineering,Calgary,Canada"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Calgary,Schulich School of Engineering,Calgary,Canada","institution_ids":["https://openalex.org/I168635309"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5020523240","display_name":"Maleknaz Nayebi","orcid":"https://orcid.org/0000-0002-2243-5721","observed_orcids":["https://orcid.org/0000-0002-2243-5721"]},"institutions":[{"id":"https://openalex.org/I192455969","display_name":"York University","ror":"https://ror.org/05fq50484","country_code":"CA","type":"education","lineage":["https://openalex.org/I192455969"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Maleknaz Nayebi","raw_affiliation_strings":["York University,Lassonde School of Engineering,Toronto,Canada"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"York University,Lassonde School of Engineering,Toronto,Canada","institution_ids":["https://openalex.org/I192455969"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100326214","display_name":"Song Wang","orcid":"https://orcid.org/0000-0003-0617-2877","observed_orcids":["https://orcid.org/0000-0003-0617-2877","https://orcid.org/0000-0003-0531-5717"]},"institutions":[{"id":"https://openalex.org/I192455969","display_name":"York University","ror":"https://ror.org/05fq50484","country_code":"CA","type":"education","lineage":["https://openalex.org/I192455969"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Song Wang","raw_affiliation_strings":["York University,Lassonde School of Engineering,Toronto,Canada"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"York University,Lassonde School of Engineering,Toronto,Canada","institution_ids":["https://openalex.org/I192455969"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5070634219","display_name":"Hadi Hemmati","orcid":"https://orcid.org/0000-0003-0204-9812","observed_orcids":["https://orcid.org/0000-0003-0204-9812"]},"institutions":[{"id":"https://openalex.org/I192455969","display_name":"York University","ror":"https://ror.org/05fq50484","country_code":"CA","type":"education","lineage":["https://openalex.org/I192455969"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Hadi Hemmati","raw_affiliation_strings":["York University,Lassonde School of Engineering,Toronto,Canada"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"York University,Lassonde School of Engineering,Toronto,Canada","institution_ids":["https://openalex.org/I192455969"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":68.5352,"has_fulltext":true,"cited_by_count":156,"citation_normalized_percentile":{"value":0.99947986,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":88,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"490","last_page":"502"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9965999722480774,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9965999722480774,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.0024999999441206455,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":9.999999747378752e-05,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/prompt-engineering","display_name":"prompt engineering","score":0.9430000185966492},{"id":"https://openalex.org/keywords/fine-tuning","display_name":"fine-tuning","score":0.9610000252723694},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.7580000162124634},{"id":"https://openalex.org/keywords/code-generation","display_name":"code generation","score":0.7110000252723694},{"id":"https://openalex.org/keywords/code-summarization","display_name":"code summarization","score":0.9539999961853027},{"id":"https://openalex.org/keywords/code-translation","display_name":"program translation","score":0.9980000257492065},{"id":"https://openalex.org/keywords/gpt-4","display_name":"GPT-4","score":0.9739999771118164},{"id":"https://openalex.org/keywords/in-context-learning","display_name":"in-context learning","score":0.8949999809265137},{"id":"https://openalex.org/keywords/human-feedback","display_name":"human feedback","score":0.7400000095367432},{"id":"https://openalex.org/keywords/codebert","display_name":"CodeBERT","score":0.6669999957084656}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5374228954315186},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.5125577449798584},{"id":"https://openalex.org/C120936955","wikidata":"https://www.wikidata.org/wiki/Q2155640","display_name":"Empirical research","level":2,"score":0.4155474603176117},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.18058839440345764},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1109/msr66628.2025.00082","doi":"https://doi.org/10.1109/msr66628.2025.00082","is_oa":false,"landing_page_url":"https://doi.org/10.1109/msr66628.2025.00082","pdf_url":null,"source":{"id":"https://openalex.org/S7407088599","display_name":"International Conference on Mining Software Repositories (MSR)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 IEEE/ACM 22nd International Conference on Mining Software Repositories (MSR)","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2310.10508","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2310.10508","pdf_url":"https://arxiv.org/pdf/2310.10508","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2310.10508","doi":"https://doi.org/10.48550/arxiv.2310.10508","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2310.10508","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2310.10508","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2310.10508","pdf_url":"https://arxiv.org/pdf/2310.10508","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4411271504.pdf","grobid_xml":"https://content.openalex.org/works/W4411271504.grobid-xml"},"referenced_works_count":95,"referenced_works":["https://openalex.org/W1556808170","https://openalex.org/W1976193075","https://openalex.org/W2101105183","https://openalex.org/W2129660502","https://openalex.org/W2167271913","https://openalex.org/W2360967250","https://openalex.org/W2511803001","https://openalex.org/W2765574772","https://openalex.org/W2794601162","https://openalex.org/W2884276923","https://openalex.org/W2890867094","https://openalex.org/W2911233971","https://openalex.org/W2949297108","https://openalex.org/W2969368867","https://openalex.org/W2981852735","https://openalex.org/W3030163527","https://openalex.org/W3034549508","https://openalex.org/W3089307846","https://openalex.org/W3098267758","https://openalex.org/W3098605233","https://openalex.org/W3119555019","https://openalex.org/W3122241445","https://openalex.org/W3126675481","https://openalex.org/W3170092793","https://openalex.org/W3176724752","https://openalex.org/W3177813494","https://openalex.org/W3183541766","https://openalex.org/W3183962691","https://openalex.org/W3185341429","https://openalex.org/W3195140297","https://openalex.org/W3198685994","https://openalex.org/W3203321135","https://openalex.org/W4221151149","https://openalex.org/W4221159404","https://openalex.org/W4221166942","https://openalex.org/W4224308101","https://openalex.org/W4225716708","https://openalex.org/W4225791314","https://openalex.org/W4226278401","https://openalex.org/W4226287673","https://openalex.org/W4235067964","https://openalex.org/W4235902765","https://openalex.org/W4250967164","https://openalex.org/W4252076394","https://openalex.org/W4284667247","https://openalex.org/W4284705844","https://openalex.org/W4286750487","https://openalex.org/W4288055447","https://openalex.org/W4313547549","https://openalex.org/W4313563813","https://openalex.org/W4322718191","https://openalex.org/W4323033814","https://openalex.org/W4376167329","https://openalex.org/W4376653844","https://openalex.org/W4380993527","https://openalex.org/W4383180571","https://openalex.org/W4384918448","https://openalex.org/W4385373622","https://openalex.org/W4385713994","https://openalex.org/W4385970045","https://openalex.org/W4386436496","https://openalex.org/W4387848774","https://openalex.org/W4387869001","https://openalex.org/W4388483492","https://openalex.org/W4388483640","https://openalex.org/W4389161785","https://openalex.org/W4389438938","https://openalex.org/W4389777735","https://openalex.org/W4391558516","https://openalex.org/W4391558518","https://openalex.org/W4391558635","https://openalex.org/W4391592188","https://openalex.org/W4392240262","https://openalex.org/W4393147129","https://openalex.org/W4394638297","https://openalex.org/W4394745253","https://openalex.org/W4394769102","https://openalex.org/W4396832043","https://openalex.org/W4399557965","https://openalex.org/W4400484796","https://openalex.org/W4400582230","https://openalex.org/W4400582609","https://openalex.org/W4400978763","https://openalex.org/W4401543487","https://openalex.org/W4402457261","https://openalex.org/W4402860127","https://openalex.org/W4403537047","https://openalex.org/W4403585905","https://openalex.org/W4403747681","https://openalex.org/W6855129433","https://openalex.org/W6855355790","https://openalex.org/W6967084484","https://openalex.org/W7133211372","https://openalex.org/W7133222210","https://openalex.org/W7133224126"],"related_works":["https://openalex.org/W7101833142","https://openalex.org/W7160854452","https://openalex.org/W4400367953","https://openalex.org/W4415261377","https://openalex.org/W4400520863","https://openalex.org/W4414230443","https://openalex.org/W4417024338","https://openalex.org/W7112409423","https://openalex.org/W4405894420","https://openalex.org/W4413074098"],"abstract_inverted_index":{"The":[0,162],"rapid":[1],"advancements":[2],"in":[3,22,58,123],"large":[4],"language":[5],"models":[6,97,131],"(LLMs)":[7],"have":[8],"greatly":[9],"expanded":[10],"the":[11,56,59,70,136],"potential":[12],"for":[13,72,144,204],"automated":[14,180,205,210],"code-related":[15,100,206],"tasks.":[16,147],"Two":[17],"primary":[18],"methodologies":[19],"are":[20],"used":[21],"this":[23,76],"domain:":[24],"prompt":[25,83,113,211],"engineering":[26,30,84,114,212],"and":[27,89,105,158],"fine-tuning.":[28],"Prompt":[29],"involves":[31],"applying":[32],"different":[33],"strategies":[34],"to":[35,179],"query":[36],"LLMs,":[37],"like":[38],"Chat-GPT,":[39],"while":[40],"fine-tuning":[41],"further":[42,218],"adapts":[43],"pre-trained":[44],"models,":[45],"such":[46],"as":[47],"CodeBERT,":[48],"by":[49,129,132],"training":[50],"them":[51],"on":[52,135],"task-specific":[53],"data.":[54],"Despite":[55],"growth":[57],"area,":[60],"there":[61],"remains":[62],"a":[63,149],"lack":[64],"of":[65],"comprehensive":[66],"comparative":[67],"analysis":[68],"between":[69],"approaches":[71],"code":[73,102,124,145],"models.":[74,120],"In":[75],"paper,":[77],"we":[78],"evaluate":[79],"GPT-4":[80,111,126,166,197],"using":[81],"three":[82,99],"strategies-basic":[85],"prompting,":[86],"in-context":[87],"learning,":[88],"taskspecific":[90],"prompting-and":[91],"compare":[92],"it":[93],"against":[94],"17":[95],"fine-tuned":[96,119],"across":[98],"tasks:":[101],"summarization,":[103],"generation,":[104,125],"translation.":[106],"Our":[107],"results":[108,143],"indicate":[109],"that":[110,165,196],"with":[112,167,198],"does":[115],"not":[116],"consistently":[117],"outperform":[118],"For":[121],"instance,":[122],"is":[127],"outperformed":[128],"finetuned":[130],"28.3%":[133],"points":[134],"MBPP":[137],"dataset.":[138],"It":[139],"also":[140],"shows":[141],"mixed":[142],"translation":[146],"Additionally,":[148],"user":[150],"study":[151,163],"was":[152],"conducted":[153],"involving":[154],"27":[155],"graduate":[156],"students":[157],"10":[159],"industry":[160],"practitioners.":[161],"revealed":[164],"conversational":[168,199],"prompts,":[169],"incorporating":[170],"human":[171,214],"feedback":[172],"during":[173,190],"interaction,":[174],"significantly":[175],"improved":[176],"performance":[177],"compared":[178],"prompting.":[181],"Participants":[182],"often":[183],"provided":[184],"explicit":[185],"instructions":[186],"or":[187],"added":[188],"context":[189],"these":[191],"interactions.":[192],"These":[193],"findings":[194],"suggest":[195],"prompting":[200],"holds":[201],"significant":[202],"promise":[203],"tasks,":[207],"whereas":[208],"fully":[209],"without":[213],"involvement":[215],"still":[216],"requires":[217],"investigation.":[219]},"counts_by_year":[{"year":2026,"cited_by_count":61},{"year":2025,"cited_by_count":64},{"year":2024,"cited_by_count":30},{"year":2023,"cited_by_count":1}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2025-10-10T00:00:00"}