{"id":"https://openalex.org/W4390976306","doi":"https://doi.org/10.1145/3641540","title":"On the Reliability and Explainability of Language Models for Program Generation","display_name":"On the Reliability and Explainability of Language Models for Program Generation","publication_year":2024,"publication_date":"2024-01-18","ids":{"openalex":"https://openalex.org/W4390976306","doi":"https://doi.org/10.1145/3641540"},"language":"en","primary_location":{"id":"doi:10.1145/3641540","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3641540","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3641540","source":{"id":"https://openalex.org/S142627899","display_name":"ACM Transactions on Software Engineering and Methodology","issn_l":"1049-331X","issn":["1049-331X","1557-7392"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Software Engineering and Methodology","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"bronze","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3641540","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100320074","display_name":"Yue Liu","orcid":"https://orcid.org/0000-0002-5593-5917"},"institutions":[{"id":"https://openalex.org/I56590836","display_name":"Monash University","ror":"https://ror.org/02bfwt286","country_code":"AU","type":"education","lineage":["https://openalex.org/I56590836"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Yue Liu","raw_affiliation_strings":["Monash University, Clayton, Australia","Monash University, Australia"],"raw_orcid":"https://orcid.org/0000-0002-5593-5917","affiliations":[{"raw_affiliation_string":"Monash University, Clayton, Australia","institution_ids":["https://openalex.org/I56590836"]},{"raw_affiliation_string":"Monash University, Australia","institution_ids":["https://openalex.org/I56590836"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5081449581","display_name":"Chakkrit Tantithamthavorn","orcid":"https://orcid.org/0000-0002-5516-9984"},"institutions":[{"id":"https://openalex.org/I56590836","display_name":"Monash University","ror":"https://ror.org/02bfwt286","country_code":"AU","type":"education","lineage":["https://openalex.org/I56590836"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Chakkrit Tantithamthavorn","raw_affiliation_strings":["Monash University, Clayton, Australia","Monash University, Australia"],"raw_orcid":"https://orcid.org/0000-0002-5516-9984","affiliations":[{"raw_affiliation_string":"Monash University, Clayton, Australia","institution_ids":["https://openalex.org/I56590836"]},{"raw_affiliation_string":"Monash University, Australia","institution_ids":["https://openalex.org/I56590836"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100622983","display_name":"Yonghui Liu","orcid":"https://orcid.org/0000-0001-7548-5100"},"institutions":[{"id":"https://openalex.org/I56590836","display_name":"Monash University","ror":"https://ror.org/02bfwt286","country_code":"AU","type":"education","lineage":["https://openalex.org/I56590836"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Yonghui Liu","raw_affiliation_strings":["Monash University, Clayton, Australia","Monash University, Australia"],"raw_orcid":"https://orcid.org/0000-0001-7548-5100","affiliations":[{"raw_affiliation_string":"Monash University, Clayton, Australia","institution_ids":["https://openalex.org/I56590836"]},{"raw_affiliation_string":"Monash University, Australia","institution_ids":["https://openalex.org/I56590836"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5106407830","display_name":"Li Li","orcid":"https://orcid.org/0000-0003-2990-1614"},"institutions":[{"id":"https://openalex.org/I82880672","display_name":"Beihang University","ror":"https://ror.org/00wk2mp56","country_code":"CN","type":"education","lineage":["https://openalex.org/I82880672"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Li Li","raw_affiliation_strings":["Beihang University, Beijing, China","Beihang University, China"],"raw_orcid":"https://orcid.org/0000-0003-2990-1614","affiliations":[{"raw_affiliation_string":"Beihang University, Beijing, China","institution_ids":["https://openalex.org/I82880672"]},{"raw_affiliation_string":"Beihang University, China","institution_ids":["https://openalex.org/I82880672"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":17.9081,"has_fulltext":true,"cited_by_count":28,"citation_normalized_percentile":{"value":0.99182186,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":100},"biblio":{"volume":"33","issue":"5","first_page":"1","last_page":"26"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12423","display_name":"Software Reliability and Analysis Research","score":0.992900013923645,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9864000082015991,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8767499923706055},{"id":"https://openalex.org/keywords/robustness","display_name":"Robustness (evolution)","score":0.5726845264434814},{"id":"https://openalex.org/keywords/code-generation","display_name":"Code generation","score":0.5364651679992676},{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.506860077381134},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4988374710083008},{"id":"https://openalex.org/keywords/trustworthiness","display_name":"Trustworthiness","score":0.4836105406284332},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.47958484292030334},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.47849005460739136},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.4757285416126251},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.4126995801925659},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.3603549599647522},{"id":"https://openalex.org/keywords/reliability-engineering","display_name":"Reliability engineering","score":0.3559502363204956}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8767499923706055},{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.5726845264434814},{"id":"https://openalex.org/C133162039","wikidata":"https://www.wikidata.org/wiki/Q1061077","display_name":"Code generation","level":3,"score":0.5364651679992676},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.506860077381134},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4988374710083008},{"id":"https://openalex.org/C153701036","wikidata":"https://www.wikidata.org/wiki/Q659974","display_name":"Trustworthiness","level":2,"score":0.4836105406284332},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.47958484292030334},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.47849005460739136},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.4757285416126251},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.4126995801925659},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3603549599647522},{"id":"https://openalex.org/C200601418","wikidata":"https://www.wikidata.org/wiki/Q2193887","display_name":"Reliability engineering","level":1,"score":0.3559502363204956},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0},{"id":"https://openalex.org/C55493867","wikidata":"https://www.wikidata.org/wiki/Q7094","display_name":"Biochemistry","level":1,"score":0.0},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.0},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.0},{"id":"https://openalex.org/C104317684","wikidata":"https://www.wikidata.org/wiki/Q7187","display_name":"Gene","level":2,"score":0.0},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.0},{"id":"https://openalex.org/C163258240","wikidata":"https://www.wikidata.org/wiki/Q25342","display_name":"Power (physics)","level":2,"score":0.0},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3641540","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3641540","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3641540","source":{"id":"https://openalex.org/S142627899","display_name":"ACM Transactions on Software Engineering and Methodology","issn_l":"1049-331X","issn":["1049-331X","1557-7392"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Software Engineering and Methodology","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1145/3641540","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3641540","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3641540","source":{"id":"https://openalex.org/S142627899","display_name":"ACM Transactions on Software Engineering and Methodology","issn_l":"1049-331X","issn":["1049-331X","1557-7392"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Software Engineering and Methodology","raw_type":"journal-article"},"sustainable_development_goals":[{"display_name":"Quality Education","score":0.5,"id":"https://metadata.un.org/sdg/4"}],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4390976306.pdf","grobid_xml":"https://content.openalex.org/works/W4390976306.grobid-xml"},"referenced_works_count":94,"referenced_works":["https://openalex.org/W1593271688","https://openalex.org/W2247864914","https://openalex.org/W2794601162","https://openalex.org/W2892181857","https://openalex.org/W2911270308","https://openalex.org/W2911282308","https://openalex.org/W2954823997","https://openalex.org/W2970971581","https://openalex.org/W2972082064","https://openalex.org/W2973529529","https://openalex.org/W2979792666","https://openalex.org/W2980282514","https://openalex.org/W2981852735","https://openalex.org/W3013745307","https://openalex.org/W3033638351","https://openalex.org/W3085380432","https://openalex.org/W3093464584","https://openalex.org/W3098605233","https://openalex.org/W3121707215","https://openalex.org/W3161997752","https://openalex.org/W3168488662","https://openalex.org/W3170092793","https://openalex.org/W3174067697","https://openalex.org/W3174750614","https://openalex.org/W3176390156","https://openalex.org/W3182156464","https://openalex.org/W3183962691","https://openalex.org/W3195421019","https://openalex.org/W3198685994","https://openalex.org/W3211801722","https://openalex.org/W4200633062","https://openalex.org/W4205601502","https://openalex.org/W4205759361","https://openalex.org/W4206756860","https://openalex.org/W4210440021","https://openalex.org/W4210499321","https://openalex.org/W4220722393","https://openalex.org/W4221143517","https://openalex.org/W4221157342","https://openalex.org/W4225640545","https://openalex.org/W4235952395","https://openalex.org/W4238596068","https://openalex.org/W4283032596","https://openalex.org/W4283751459","https://openalex.org/W4284664028","https://openalex.org/W4284709233","https://openalex.org/W4284710241","https://openalex.org/W4285069927","https://openalex.org/W4285490400","https://openalex.org/W4285490465","https://openalex.org/W4285583181","https://openalex.org/W4288055447","https://openalex.org/W4290855557","https://openalex.org/W4297847088","https://openalex.org/W4308641647","https://openalex.org/W4308641648","https://openalex.org/W4308643013","https://openalex.org/W4308643015","https://openalex.org/W4308643994","https://openalex.org/W4312290548","https://openalex.org/W4312372164","https://openalex.org/W4312436517","https://openalex.org/W4313547604","https://openalex.org/W4319049672","https://openalex.org/W4320854090","https://openalex.org/W4321065626","https://openalex.org/W4327673267","https://openalex.org/W4367059303","https://openalex.org/W4367059627","https://openalex.org/W4367061230","https://openalex.org/W4367672983","https://openalex.org/W4376606821","https://openalex.org/W4376606857","https://openalex.org/W4376652621","https://openalex.org/W4378713557","https://openalex.org/W4384155563","https://openalex.org/W4384155620","https://openalex.org/W4384345639","https://openalex.org/W4384347367","https://openalex.org/W4385261783","https://openalex.org/W4385884967","https://openalex.org/W4386081573","https://openalex.org/W4386614573","https://openalex.org/W4387363888","https://openalex.org/W4387427818","https://openalex.org/W4388638461","https://openalex.org/W4389519352","https://openalex.org/W4389988584","https://openalex.org/W4393406994","https://openalex.org/W4394652731","https://openalex.org/W6739901393","https://openalex.org/W6748774801","https://openalex.org/W6754149038","https://openalex.org/W6778883912"],"related_works":["https://openalex.org/W2378211422","https://openalex.org/W2745001401","https://openalex.org/W4321353415","https://openalex.org/W2130974462","https://openalex.org/W972276598","https://openalex.org/W4246352526","https://openalex.org/W2028665553","https://openalex.org/W4230315250","https://openalex.org/W2086519370","https://openalex.org/W2087343574"],"abstract_inverted_index":{"Recent":[0],"studies":[1],"have":[2,28],"adopted":[3],"pre-trained":[4],"language":[5,24,101,163],"models,":[6,50],"such":[7],"as":[8],"CodeT5":[9],"and":[10,21,31,82,85,111,169,187,195],"CodeGPT,":[11],"for":[12,69,208],"automated":[13,70,114,198],"program":[14,71,115,199],"generation":[15,116,200],"tasks":[16],"like":[17],"code":[18,58,134,167],"generation,":[19],"repair,":[20],"translation.":[22],"Numerous":[23],"model":[25,80],"based":[26],"approaches":[27,124,140,186],"been":[29],"proposed":[30],"evaluated":[32],"on":[33,103],"various":[34,160],"benchmark":[35],"datasets,":[36],"demonstrating":[37],"promising":[38],"performance.":[39],"However,":[40],"there":[41],"is":[42,76],"still":[43],"uncertainty":[44],"about":[45],"the":[46,109,127,133,193],"reliability":[47,84,194],"of":[48,98,113,197],"these":[49,65,89],"particularly":[51],"their":[52],"realistic":[53],"ability":[54],"to":[55,78,107,125,132,177,191],"consistently":[56],"transform":[57],"sequences.":[59,181],"This":[60],"raises":[61],"a":[62,94],"question:":[63],"are":[64,189],"techniques":[66],"sufficiently":[67],"trustworthy":[68],"generation?":[72],"Consequently,":[73],"further":[74,119],"research":[75,90],"needed":[77],"understand":[79],"logic":[81],"assess":[83],"explainability.":[86],"To":[87],"bridge":[88],"gaps,":[91],"we":[92],"conduct":[93],"thorough":[95],"empirical":[96],"study":[97],"eight":[99],"popular":[100],"models":[102,164],"five":[104],"representative":[105],"datasets":[106],"determine":[108],"capabilities":[110],"limitations":[112],"approaches.":[117],"We":[118,136],"employ":[120],"advanced":[121],"explainable":[122],"AI":[123],"highlight":[126],"tokens":[128],"that":[129,138],"significantly":[130],"contribute":[131],"transformation.":[135],"discover":[137],"state-of-the-art":[139],"suffer":[141],"from":[142,147],"inappropriate":[143],"performance":[144],"evaluation":[145,185],"stemming":[146],"severe":[148],"data":[149],"duplication,":[150],"causing":[151],"overoptimistic":[152],"results.":[153],"Our":[154,203],"explainability":[155,196],"analysis":[156],"reveals":[157],"that,":[158],"in":[159,179],"experimental":[161],"scenarios,":[162],"can":[165],"recognize":[166],"grammar":[168],"structural":[170],"information,":[171],"but":[172],"they":[173],"exhibit":[174],"limited":[175],"robustness":[176],"changes":[178],"input":[180],"Overall,":[182],"more":[183],"rigorous":[184],"benchmarks":[188],"critical":[190],"enhance":[192],"moving":[201],"forward.":[202],"findings":[204],"provide":[205],"important":[206],"guidelines":[207],"this":[209],"goal.":[210]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":14},{"year":2024,"cited_by_count":11}],"updated_date":"2026-05-21T06:26:12.895304","created_date":"2025-10-10T00:00:00"}