{"id":"https://openalex.org/W4381252103","doi":"https://doi.org/10.48550/arxiv.2306.09896","title":"Is Self-Repair a Silver Bullet for Code Generation?","display_name":"Is Self-Repair a Silver Bullet for Code Generation?","publication_year":2023,"publication_date":"2023-06-16","ids":{"openalex":"https://openalex.org/W4381252103","doi":"https://doi.org/10.48550/arxiv.2306.09896"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2306.09896","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2306.09896","pdf_url":"https://arxiv.org/pdf/2306.09896","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2306.09896","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5035758082","display_name":"Theo Olausson","orcid":"https://orcid.org/0000-0001-6653-2227","observed_orcids":["https://orcid.org/0000-0001-6653-2227"]},"institutions":[{"id":"https://openalex.org/I4405258931","display_name":"MIT Computer Science and Artificial Intelligence Laboratory","ror":"https://ror.org/01e85r662","country_code":"US","type":"facility","lineage":["https://openalex.org/I4405258931","https://openalex.org/I63966007"]},{"id":"https://openalex.org/I63966007","display_name":"Massachusetts Institute of Technology","ror":"https://ror.org/042nb2s44","country_code":"US","type":"education","lineage":["https://openalex.org/I63966007"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Olausson, Theo X.","raw_affiliation_strings":["MIT CSAIL"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"MIT CSAIL","institution_ids":["https://openalex.org/I4405258931","https://openalex.org/I63966007"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5020203532","display_name":"Jeevana Priya Inala","orcid":"https://orcid.org/0000-0003-1843-589X","observed_orcids":["https://orcid.org/0000-0003-1843-589X"]},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Inala, Jeevana Priya","raw_affiliation_strings":["Microsoft Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Research","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100367824","display_name":"Chenglong Wang","orcid":"https://orcid.org/0000-0003-2262-4167","observed_orcids":["https://orcid.org/0000-0003-2262-4167"]},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Wang, Chenglong","raw_affiliation_strings":["Microsoft Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Research","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5114910293","display_name":"Jianfeng Gao","orcid":"https://orcid.org/0000-0002-5702-6143","observed_orcids":["https://orcid.org/0000-0002-5702-6143"]},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Gao, Jianfeng","raw_affiliation_strings":["Microsoft Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Research","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5010786661","display_name":"Armando Solar-Lezama","orcid":"https://orcid.org/0000-0001-7604-8252","observed_orcids":["https://orcid.org/0000-0001-7604-8252"]},"institutions":[{"id":"https://openalex.org/I4405258931","display_name":"MIT Computer Science and Artificial Intelligence Laboratory","ror":"https://ror.org/01e85r662","country_code":"US","type":"facility","lineage":["https://openalex.org/I4405258931","https://openalex.org/I63966007"]},{"id":"https://openalex.org/I63966007","display_name":"Massachusetts Institute of Technology","ror":"https://ror.org/042nb2s44","country_code":"US","type":"education","lineage":["https://openalex.org/I63966007"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Solar-Lezama, Armando","raw_affiliation_strings":["MIT CSAIL"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"MIT CSAIL","institution_ids":["https://openalex.org/I4405258931","https://openalex.org/I63966007"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":13,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9800000190734863,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9800000190734863,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.014299999922513962,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10533","display_name":"Teaching and Learning Programming","score":0.001500000013038516,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/self-healing","display_name":"self-healing","score":0.9660000205039978},{"id":"https://openalex.org/keywords/code-generation","display_name":"code generation","score":0.7710000276565552},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.902999997138977},{"id":"https://openalex.org/keywords/debugging","display_name":"debugging","score":0.5879999995231628},{"id":"https://openalex.org/keywords/code-llama","display_name":"CodeLlama","score":0.7689999938011169},{"id":"https://openalex.org/keywords/gpt-4","display_name":"GPT-4","score":0.9359999895095825},{"id":"https://openalex.org/keywords/gpt-3-5","display_name":"GPT-3.5","score":0.8769999742507935},{"id":"https://openalex.org/keywords/humaneval","display_name":"HumanEval","score":0.9269999861717224},{"id":"https://openalex.org/keywords/repair-costs","display_name":"repair costs","score":0.5569999814033508}],"concepts":[{"id":"https://openalex.org/C2776088982","wikidata":"https://www.wikidata.org/wiki/Q841402","display_name":"Silver bullet","level":2,"score":0.9080150127410889},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.5452159643173218},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.398536741733551},{"id":"https://openalex.org/C192562407","wikidata":"https://www.wikidata.org/wiki/Q228736","display_name":"Materials science","level":0,"score":0.18653425574302673},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.16036194562911987},{"id":"https://openalex.org/C191897082","wikidata":"https://www.wikidata.org/wiki/Q11467","display_name":"Metallurgy","level":1,"score":0.06566479802131653},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2306.09896","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2306.09896","pdf_url":"https://arxiv.org/pdf/2306.09896","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2306.09896","doi":"https://doi.org/10.48550/arxiv.2306.09896","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2306.09896","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2306.09896","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2306.09896","pdf_url":"https://arxiv.org/pdf/2306.09896","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G7588888253","display_name":"PPoSS: LARGE: Intel: Combining Learning and Formal Verification for Scalable Machine Programming (ScaMP)","funder_award_id":"2217064","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320307102","display_name":"Intel Corporation","ror":"https://ror.org/01ek73717"},{"id":"https://openalex.org/F4320308943","display_name":"Microsoft Research","ror":"https://ror.org/00d0nc645"},{"id":"https://openalex.org/F4320332180","display_name":"Defense Advanced Research Projects Agency","ror":"https://ror.org/02caytj08"},{"id":"https://openalex.org/F4320332815","display_name":"Advanced Research Projects Agency","ror":"https://ror.org/02caytj08"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4381252103.pdf","grobid_xml":"https://content.openalex.org/works/W4381252103.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W7154279859","https://openalex.org/W4408443504","https://openalex.org/W7138857416","https://openalex.org/W4365205411","https://openalex.org/W4416042598","https://openalex.org/W7167644418","https://openalex.org/W4417248053","https://openalex.org/W7215061966","https://openalex.org/W4402670831","https://openalex.org/W4396214504"],"abstract_inverted_index":{"Large":[0],"language":[1],"models":[2],"have":[3,51],"shown":[4],"remarkable":[5],"aptitude":[6],"in":[7,19,39,54,56,162],"code":[8,28],"generation,":[9],"but":[10],"still":[11,180],"struggle":[12],"to":[13,36,76,133,144],"perform":[14,77],"complex":[15],"tasks.":[16],"Self-repair":[17],"--":[18,29],"which":[20,163],"the":[21,90,111,130,147,150,176],"model":[22,143],"debugs":[23],"and":[24,73,84,113],"repairs":[25],"its":[26,44,59,137],"own":[27,138],"has":[30],"recently":[31],"become":[32],"a":[33,106,141,159],"popular":[34],"way":[35],"boost":[37,146],"performance":[38,100,156],"these":[40],"settings.":[41],"However,":[42],"despite":[43],"increasing":[45],"popularity,":[46],"existing":[47],"studies":[48],"of":[49,92,110,149],"self-repair":[50,78,126,179],"been":[52],"limited":[53],"scope;":[55],"many":[57],"settings,":[58],"efficacy":[60],"thus":[61],"remains":[62],"poorly":[63],"understood.":[64],"In":[65],"this":[66,123],"paper,":[67],"we":[68,152,164],"analyze":[69],"Code":[70],"Llama,":[71],"GPT-3.5":[72],"GPT-4's":[74],"ability":[75,132],"on":[79,136],"problems":[80],"taken":[81,97],"from":[82,169],"HumanEval":[83],"APPS.":[85],"We":[86,120],"find":[87],"that":[88,122,173],"when":[89],"cost":[91],"carrying":[93],"out":[94],"repair":[95],"is":[96,124,127],"into":[98],"account,":[99],"gains":[101],"are":[102,114],"often":[103],"modest,":[104],"vary":[105],"lot":[107],"between":[108],"subsets":[109],"data,":[112],"sometimes":[115],"not":[116],"present":[117],"at":[118],"all.":[119],"hypothesize":[121],"because":[125],"bottlenecked":[128],"by":[129],"model's":[131],"provide":[134,165],"feedback":[135,168],"code;":[139],"using":[140],"stronger":[142],"artificially":[145],"quality":[148],"feedback,":[151],"observe":[153],"substantially":[154],"larger":[155],"gains.":[157],"Similarly,":[158],"small-scale":[160],"study":[161],"GPT-4":[166],"with":[167,188],"human":[170],"participants":[171],"suggests":[172],"even":[174],"for":[175],"strongest":[177],"models,":[178],"lags":[181],"far":[182],"behind":[183],"what":[184],"can":[185],"be":[186],"achieved":[187],"human-level":[189],"debugging.":[190]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":4},{"year":2024,"cited_by_count":4},{"year":2023,"cited_by_count":2}],"updated_date":"2026-10-16T09:59:22.235173","created_date":"2023-06-20T00:00:00"}