{"id":"https://openalex.org/W3212260201","doi":"https://doi.org/10.48550/arxiv.2111.03922","title":"Automatic Program Repair with OpenAI's Codex: Evaluating QuixBugs","display_name":"Automatic Program Repair with OpenAI's Codex: Evaluating QuixBugs","publication_year":2021,"publication_date":"2021-11-06","ids":{"openalex":"https://openalex.org/W3212260201","doi":"https://doi.org/10.48550/arxiv.2111.03922","mag":"3212260201"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2111.03922","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2111.03922","pdf_url":"https://arxiv.org/pdf/2111.03922","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2111.03922","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5027995569","display_name":"Julian Aron Prenner","orcid":"https://orcid.org/0000-0003-4673-271X","observed_orcids":["https://orcid.org/0000-0003-4673-271X"]},"institutions":[{"id":"https://openalex.org/I171543936","display_name":"Free University of Bozen-Bolzano","ror":"https://ror.org/012ajp527","country_code":"IT","type":"education","lineage":["https://openalex.org/I171543936"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Prenner, Julian Aron","raw_affiliation_strings":["Free University of Bozen-Bolzano"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Free University of Bozen-Bolzano","institution_ids":["https://openalex.org/I171543936"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5031712876","display_name":"Romain Robbes","orcid":"https://orcid.org/0000-0003-4569-6868","observed_orcids":["https://orcid.org/0000-0003-4569-6868"]},"institutions":[{"id":"https://openalex.org/I171543936","display_name":"Free University of Bozen-Bolzano","ror":"https://ror.org/012ajp527","country_code":"IT","type":"education","lineage":["https://openalex.org/I171543936"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Robbes, Romain","raw_affiliation_strings":["Free University of Bozen-Bolzano"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Free University of Bozen-Bolzano","institution_ids":["https://openalex.org/I171543936"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I171543936"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":72,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9208999872207642,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9208999872207642,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.07779999822378159,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11975","display_name":"Evolutionary Algorithms and Applications","score":9.999999747378752e-05,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/automated-program-repair","display_name":"automated program repair","score":0.8650000095367432},{"id":"https://openalex.org/keywords/codex","display_name":"codices","score":0.796999990940094},{"id":"https://openalex.org/keywords/gpt-3","display_name":"GPT-3","score":0.8550000190734863},{"id":"https://openalex.org/keywords/bug-localization","display_name":"bug localization","score":0.8479999899864197},{"id":"https://openalex.org/keywords/bug-fixing","display_name":"bug fixes","score":0.8700000047683716},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.7160000205039978},{"id":"https://openalex.org/keywords/python","display_name":"Python","score":0.6430000066757202},{"id":"https://openalex.org/keywords/java","display_name":"Java","score":0.8069999814033508}],"concepts":[{"id":"https://openalex.org/C519991488","wikidata":"https://www.wikidata.org/wiki/Q28865","display_name":"Python (programming language)","level":2,"score":0.8780860900878906},{"id":"https://openalex.org/C548217200","wikidata":"https://www.wikidata.org/wiki/Q251","display_name":"Java","level":2,"score":0.8089379072189331},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.7518587112426758},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7390124797821045},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.595346212387085},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.43524667620658875},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.38523441553115845},{"id":"https://openalex.org/C58640448","wikidata":"https://www.wikidata.org/wiki/Q42515","display_name":"Cartography","level":1,"score":0.07594096660614014},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"pmh:oai:arXiv.org:2111.03922","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2111.03922","pdf_url":"https://arxiv.org/pdf/2111.03922","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"mag:3212260201","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/pdf/2111.03922.pdf","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null},{"id":"doi:10.48550/arxiv.2111.03922","doi":"https://doi.org/10.48550/arxiv.2111.03922","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2111.03922","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2111.03922","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2111.03922","pdf_url":"https://arxiv.org/pdf/2111.03922","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3212260201.pdf","grobid_xml":"https://content.openalex.org/works/W3212260201.grobid-xml"},"referenced_works_count":23,"referenced_works":["https://openalex.org/W2076719273","https://openalex.org/W2145373440","https://openalex.org/W2148357053","https://openalex.org/W2156723666","https://openalex.org/W2626778328","https://openalex.org/W2762550985","https://openalex.org/W2767766265","https://openalex.org/W2767951593","https://openalex.org/W2896457183","https://openalex.org/W2907705732","https://openalex.org/W2908354350","https://openalex.org/W2952903800","https://openalex.org/W2963341956","https://openalex.org/W2963909831","https://openalex.org/W2996248296","https://openalex.org/W3001279689","https://openalex.org/W3030163527","https://openalex.org/W3043761819","https://openalex.org/W3098903812","https://openalex.org/W3101193405","https://openalex.org/W3134686229","https://openalex.org/W3161282429","https://openalex.org/W3177813494"],"related_works":["https://openalex.org/W4312970618","https://openalex.org/W4396765406","https://openalex.org/W4312962198","https://openalex.org/W4385302156","https://openalex.org/W7111505947","https://openalex.org/W4388095126","https://openalex.org/W4402457546","https://openalex.org/W4384345745","https://openalex.org/W7152933346","https://openalex.org/W4396214523"],"abstract_inverted_index":{"OpenAI's":[0],"Codex,":[1],"a":[2,8,21,57],"GPT-3":[3],"like":[4],"model":[5],"trained":[6,90],"on":[7],"large":[9],"code":[10,30],"corpus,":[11],"has":[12],"made":[13],"headlines":[14],"in":[15,38,62,79],"and":[16,35,54,82,97],"outside":[17],"of":[18,28,59,65,102],"academia.":[19],"Given":[20],"short":[22],"user-provided":[23],"description,":[24],"it":[25],"is":[26,50,94,112],"capable":[27],"synthesizing":[29],"snippets":[31],"that":[32,110],"are":[33],"syntactically":[34],"semantically":[36],"valid":[37],"most":[39],"cases.":[40],"In":[41],"this":[42],"work,":[43],"we":[44],"want":[45],"to":[46,52],"investigate":[47],"whether":[48],"Codex":[49,93,111],"able":[51],"localize":[53],"fix":[55],"bugs,":[56],"task":[58],"central":[60],"interest":[61],"the":[63,73,103],"field":[64],"automated":[66],"program":[67],"repair.":[68],"Our":[69,106],"initial":[70],"evaluation":[71],"uses":[72],"multi-language":[74],"QuixBugs":[75],"benchmark":[76],"(40":[77],"bugs":[78],"both":[80],"Python":[81,118],"Java).":[83],"We":[84],"find":[85],"that,":[86],"despite":[87],"not":[88],"being":[89],"for":[91],"APR,":[92],"surprisingly":[95],"effective,":[96],"competitive":[98],"with":[99],"recent":[100],"state":[101],"art":[104],"techniques.":[105],"results":[107],"also":[108],"show":[109],"slightly":[113],"more":[114],"successful":[115],"at":[116],"repairing":[117],"than":[119],"Java.":[120]},"counts_by_year":[{"year":2026,"cited_by_count":9},{"year":2025,"cited_by_count":11},{"year":2024,"cited_by_count":21},{"year":2023,"cited_by_count":25},{"year":2022,"cited_by_count":6}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2021-11-22T00:00:00"}