{"id":"https://openalex.org/W4400485045","doi":"https://doi.org/10.1145/3664646.3664772","title":"Chain of Targeted Verification Questions to Improve the Reliability of Code Generated by LLMs","display_name":"Chain of Targeted Verification Questions to Improve the Reliability of Code Generated by LLMs","publication_year":2024,"publication_date":"2024-07-10","ids":{"openalex":"https://openalex.org/W4400485045","doi":"https://doi.org/10.1145/3664646.3664772"},"language":"en","primary_location":{"id":"doi:10.1145/3664646.3664772","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3664646.3664772","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 1st ACM International Conference on AI-Powered Software","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5098838398","display_name":"Sylvain Kouemo Ngassom","orcid":"https://orcid.org/0009-0000-3215-8863"},"institutions":[{"id":"https://openalex.org/I45683168","display_name":"Polytechnique Montr\u00e9al","ror":"https://ror.org/05f8d4e86","country_code":"CA","type":"education","lineage":["https://openalex.org/I45683168"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Sylvain Kouemo Ngassom","raw_affiliation_strings":["Polytechnique Montr\u00e9al, Montreal, Canada"],"raw_orcid":"https://orcid.org/0009-0000-3215-8863","affiliations":[{"raw_affiliation_string":"Polytechnique Montr\u00e9al, Montreal, Canada","institution_ids":["https://openalex.org/I45683168"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5065291839","display_name":"Arghavan Moradi Dakhel","orcid":"https://orcid.org/0000-0003-1900-2850"},"institutions":[{"id":"https://openalex.org/I45683168","display_name":"Polytechnique Montr\u00e9al","ror":"https://ror.org/05f8d4e86","country_code":"CA","type":"education","lineage":["https://openalex.org/I45683168"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Arghavan Moradi Dakhel","raw_affiliation_strings":["Polytechnique Montr\u00e9al, Montreal, Canada"],"raw_orcid":"https://orcid.org/0000-0003-1900-2850","affiliations":[{"raw_affiliation_string":"Polytechnique Montr\u00e9al, Montreal, Canada","institution_ids":["https://openalex.org/I45683168"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5059516159","display_name":"Florian Tambon","orcid":"https://orcid.org/0000-0001-5593-9400"},"institutions":[{"id":"https://openalex.org/I45683168","display_name":"Polytechnique Montr\u00e9al","ror":"https://ror.org/05f8d4e86","country_code":"CA","type":"education","lineage":["https://openalex.org/I45683168"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Florian Tambon","raw_affiliation_strings":["Polytechnique Montr\u00e9al, Montreal, Canada"],"raw_orcid":"https://orcid.org/0000-0001-5593-9400","affiliations":[{"raw_affiliation_string":"Polytechnique Montr\u00e9al, Montreal, Canada","institution_ids":["https://openalex.org/I45683168"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5071052367","display_name":"Foutse Khomh","orcid":"https://orcid.org/0000-0002-5704-4173"},"institutions":[{"id":"https://openalex.org/I45683168","display_name":"Polytechnique Montr\u00e9al","ror":"https://ror.org/05f8d4e86","country_code":"CA","type":"education","lineage":["https://openalex.org/I45683168"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Foutse Khomh","raw_affiliation_strings":["Polytechnique Montr\u00e9al, Montreal, Canada"],"raw_orcid":"https://orcid.org/0000-0002-5704-4173","affiliations":[{"raw_affiliation_string":"Polytechnique Montr\u00e9al, Montreal, Canada","institution_ids":["https://openalex.org/I45683168"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":1.7246,"has_fulltext":false,"cited_by_count":4,"citation_normalized_percentile":{"value":0.85879677,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"122","last_page":"130"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9959999918937683,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9959999918937683,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12423","display_name":"Software Reliability and Analysis Research","score":0.9872999787330627,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9672999978065491,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.6563946008682251},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6059812307357788},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.5381755232810974},{"id":"https://openalex.org/keywords/chain","display_name":"Chain (unit)","score":0.46428099274635315},{"id":"https://openalex.org/keywords/reliability-engineering","display_name":"Reliability engineering","score":0.40936145186424255},{"id":"https://openalex.org/keywords/risk-analysis","display_name":"Risk analysis (engineering)","score":0.35185176134109497},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.23437482118606567},{"id":"https://openalex.org/keywords/business","display_name":"Business","score":0.16792863607406616},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.16532039642333984}],"concepts":[{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.6563946008682251},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6059812307357788},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.5381755232810974},{"id":"https://openalex.org/C199185054","wikidata":"https://www.wikidata.org/wiki/Q552299","display_name":"Chain (unit)","level":2,"score":0.46428099274635315},{"id":"https://openalex.org/C200601418","wikidata":"https://www.wikidata.org/wiki/Q2193887","display_name":"Reliability engineering","level":1,"score":0.40936145186424255},{"id":"https://openalex.org/C112930515","wikidata":"https://www.wikidata.org/wiki/Q4389547","display_name":"Risk analysis (engineering)","level":1,"score":0.35185176134109497},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.23437482118606567},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.16792863607406616},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.16532039642333984},{"id":"https://openalex.org/C1276947","wikidata":"https://www.wikidata.org/wiki/Q333","display_name":"Astronomy","level":1,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C163258240","wikidata":"https://www.wikidata.org/wiki/Q25342","display_name":"Power (physics)","level":2,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3664646.3664772","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3664646.3664772","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 1st ACM International Conference on AI-Powered Software","raw_type":"proceedings-article"},{"id":"pmh:oai:publications.polymtl.ca:59068","is_oa":false,"landing_page_url":"https://publications.polymtl.ca/59068/","pdf_url":null,"source":{"id":"https://openalex.org/S4306401013","display_name":"PolyPublie (\u00c9cole Polytechnique de Montr\u00e9al)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I45683168","host_organization_name":"Polytechnique Montr\u00e9al","host_organization_lineage":["https://openalex.org/I45683168"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Communication de conf\u00e9rence"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":19,"referenced_works":["https://openalex.org/W2922250230","https://openalex.org/W3023394394","https://openalex.org/W3175279244","https://openalex.org/W4225108562","https://openalex.org/W4366735548","https://openalex.org/W4367595583","https://openalex.org/W4367672983","https://openalex.org/W4382239980","https://openalex.org/W4384345639","https://openalex.org/W4384345708","https://openalex.org/W4384345745","https://openalex.org/W4388483039","https://openalex.org/W4389158474","https://openalex.org/W4391282616","https://openalex.org/W4391558462","https://openalex.org/W4391584367","https://openalex.org/W4401208776","https://openalex.org/W4401395592","https://openalex.org/W4402670429"],"related_works":["https://openalex.org/W2033512842","https://openalex.org/W4322734194","https://openalex.org/W4233600955","https://openalex.org/W2913665393","https://openalex.org/W2369695847","https://openalex.org/W3005535424","https://openalex.org/W2994319598","https://openalex.org/W2047067935","https://openalex.org/W1607054433","https://openalex.org/W2110842462"],"abstract_inverted_index":{"LLM-based":[0,84],"assistants,":[1],"such":[2,72],"as":[3,27],"GitHub":[4],"Copilot":[5],"and":[6,44,62,81,96,140,212,248],"ChatGPT,":[7],"have":[8,180],"the":[9,92,102,122,131,142,161,170,176,181,206,209,213,223,236,242,250],"potential":[10,158,182,202],"to":[11,26,41,60,75,94,156,183,199,246,256],"generate":[12,42],"code":[13,43,54,74,125,243,254],"that":[14,53,227],"fulfills":[15],"a":[16,21,28,116],"programming":[17,220],"task":[18,69],"described":[19],"in":[20,68,83,101,141,192,222,241],"natural":[22],"language":[23],"description,":[24],"referred":[25],"prompt.":[29],"The":[30],"widespread":[31],"accessibility":[32],"of":[33,124,133,144,175,187,238,252],"these":[34,201],"assistants":[35],"enables":[36],"users":[37,76],"with":[38,208],"diverse":[39],"backgrounds":[40],"integrate":[45],"it":[46],"into":[47],"software":[48],"projects.":[49],"However,":[50],"studies":[51],"show":[52],"generated":[55,126],"by":[56,91,127,129,204,234],"LLMs":[57,128],"is":[58,149],"prone":[59],"bugs":[61,134,159,203],"may":[63],"miss":[64],"various":[65,167],"corner":[66],"cases":[67,108],"specifications.":[70],"Presenting":[71],"buggy":[73],"can":[77],"impact":[78],"their":[79],"reliability":[80,123],"trust":[82],"assistants.":[85],"Moreover,":[86],"significant":[87],"efforts":[88],"are":[89,109],"required":[90],"user":[93],"detect":[95],"repair":[97,200],"any":[98],"bug":[99,188],"present":[100],"code,":[103,178],"especially":[104],"if":[105],"no":[106],"test":[107,145],"available.":[110],"In":[111],"this":[112],"study,":[113],"we":[114],"propose":[115],"self-refinement":[117],"method":[118,197,230],"aimed":[119],"at":[120],"improving":[121,249],"minimizing":[130],"number":[132,237,251],"before":[135],"execution,":[136],"without":[137],"human":[138],"intervention,":[139],"absence":[143],"cases.":[146],"Our":[147,216],"approach":[148],"based":[150,218],"on":[151,219],"targeted":[152,210,239],"Verification":[153],"Questions":[154],"(VQs)":[155],"identify":[157],"within":[160,169],"initial":[162,177,214],"code.":[163,194,215],"These":[164],"VQs":[165,211],"target":[166],"nodes":[168],"Abstract":[171],"Syntax":[172],"Tree":[173],"(AST)":[174],"which":[179],"trigger":[184],"specific":[185],"types":[186],"patterns":[189],"commonly":[190],"found":[191],"LLM-generated":[193],"Finally,":[195],"our":[196,228],"attempts":[198],"re-prompting":[205],"LLM":[207],"evaluation,":[217],"tasks":[221],"CoderEval":[224],"dataset,":[225],"demonstrates":[226],"proposed":[229],"outperforms":[231],"state-of-the-art":[232],"methods":[233],"decreasing":[235],"errors":[240],"between":[244],"21%":[245],"62%":[247],"executable":[253],"instances":[255],"13%.":[257]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":1}],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2024-07-11T00:00:00"}