{"id":"https://openalex.org/W4390189975","doi":"https://doi.org/10.1109/milcom58377.2023.10356332","title":"Counterexample Guided Inductive Synthesis Using Large Language Models and Satisfiability Solving","display_name":"Counterexample Guided Inductive Synthesis Using Large Language Models and Satisfiability Solving","publication_year":2023,"publication_date":"2023-10-30","ids":{"openalex":"https://openalex.org/W4390189975","doi":"https://doi.org/10.1109/milcom58377.2023.10356332"},"language":"en","primary_location":{"id":"doi:10.1109/milcom58377.2023.10356332","is_oa":false,"landing_page_url":"https://doi.org/10.1109/milcom58377.2023.10356332","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"MILCOM 2023 - 2023 IEEE Military Communications Conference (MILCOM)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5075978538","display_name":"Sumit Kumar Jha","orcid":"https://orcid.org/0000-0003-0354-2940"},"institutions":[{"id":"https://openalex.org/I1298353152","display_name":"SRI International","ror":"https://ror.org/05s570m15","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I1298353152"]},{"id":"https://openalex.org/I19700959","display_name":"Florida International University","ror":"https://ror.org/02gz6gg07","country_code":"US","type":"education","lineage":["https://openalex.org/I19700959"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Sumit Kumar Jha","raw_affiliation_strings":["Computer Science Laboratory SRI International","Florida International University,Computer Science Department","Computer Science Department, Florida International University"],"affiliations":[{"raw_affiliation_string":"Computer Science Laboratory SRI International","institution_ids":["https://openalex.org/I1298353152"]},{"raw_affiliation_string":"Florida International University,Computer Science Department","institution_ids":["https://openalex.org/I19700959"]},{"raw_affiliation_string":"Computer Science Department, Florida International University","institution_ids":["https://openalex.org/I19700959"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5035902535","display_name":"Susmit Jha","orcid":"https://orcid.org/0000-0001-5983-9095"},"institutions":[{"id":"https://openalex.org/I1298353152","display_name":"SRI International","ror":"https://ror.org/05s570m15","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I1298353152"]},{"id":"https://openalex.org/I19700959","display_name":"Florida International University","ror":"https://ror.org/02gz6gg07","country_code":"US","type":"education","lineage":["https://openalex.org/I19700959"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Susmit Jha","raw_affiliation_strings":["Computer Science Laboratory SRI International","Florida International University,Computer Science Department"],"affiliations":[{"raw_affiliation_string":"Computer Science Laboratory SRI International","institution_ids":["https://openalex.org/I1298353152"]},{"raw_affiliation_string":"Florida International University,Computer Science Department","institution_ids":["https://openalex.org/I19700959"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5003908930","display_name":"Patrick Lincoln","orcid":null},"institutions":[{"id":"https://openalex.org/I1298353152","display_name":"SRI International","ror":"https://ror.org/05s570m15","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I1298353152"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Patrick Lincoln","raw_affiliation_strings":["Computer Science Laboratory SRI International"],"affiliations":[{"raw_affiliation_string":"Computer Science Laboratory SRI International","institution_ids":["https://openalex.org/I1298353152"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5032194186","display_name":"Nathaniel D. Bastian","orcid":"https://orcid.org/0000-0001-9957-2778"},"institutions":[{"id":"https://openalex.org/I4210152400","display_name":"Military Medical Academy","ror":"https://ror.org/04szvwj50","country_code":"EG","type":"healthcare","lineage":["https://openalex.org/I4210152400"]},{"id":"https://openalex.org/I192545095","display_name":"United States Military Academy","ror":"https://ror.org/01jepya76","country_code":"US","type":"education","lineage":["https://openalex.org/I1304082316","https://openalex.org/I1330347796","https://openalex.org/I192545095","https://openalex.org/I4210088792"]}],"countries":["EG","US"],"is_corresponding":false,"raw_author_name":"Nathaniel D. Bastian","raw_affiliation_strings":["United States Military Academy,Army Cyber Institute","Army Cyber Institute, United States Military Academy"],"affiliations":[{"raw_affiliation_string":"United States Military Academy,Army Cyber Institute","institution_ids":["https://openalex.org/I4210152400"]},{"raw_affiliation_string":"Army Cyber Institute, United States Military Academy","institution_ids":["https://openalex.org/I192545095"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5067109059","display_name":"Alvaro Velasquez","orcid":"https://orcid.org/0000-0001-6757-105X"},"institutions":[{"id":"https://openalex.org/I188538660","display_name":"University of Colorado Boulder","ror":"https://ror.org/02ttsq026","country_code":"US","type":"education","lineage":["https://openalex.org/I188538660"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Alvaro Velasquez","raw_affiliation_strings":["University of Colorado,Department of Computer Science,Boulder","Department of Computer Science, University of Colorado, Boulder"],"affiliations":[{"raw_affiliation_string":"University of Colorado,Department of Computer Science,Boulder","institution_ids":["https://openalex.org/I188538660"]},{"raw_affiliation_string":"Department of Computer Science, University of Colorado, Boulder","institution_ids":["https://openalex.org/I188538660"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5014892277","display_name":"Rickard Ewetz","orcid":"https://orcid.org/0000-0002-4183-6926"},"institutions":[{"id":"https://openalex.org/I106165777","display_name":"University of Central Florida","ror":"https://ror.org/036nfer12","country_code":"US","type":"education","lineage":["https://openalex.org/I106165777"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Rickard Ewetz","raw_affiliation_strings":["University of Central Florida,Electrical and Computer Engineering","Electrical and Computer Engineering, University of Central Florida"],"affiliations":[{"raw_affiliation_string":"University of Central Florida,Electrical and Computer Engineering","institution_ids":["https://openalex.org/I106165777"]},{"raw_affiliation_string":"Electrical and Computer Engineering, University of Central Florida","institution_ids":["https://openalex.org/I106165777"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5049017939","display_name":"Sandeep Neema","orcid":"https://orcid.org/0000-0002-9781-3619"},"institutions":[{"id":"https://openalex.org/I200719446","display_name":"Vanderbilt University","ror":"https://ror.org/02vm5rt34","country_code":"US","type":"education","lineage":["https://openalex.org/I200719446"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Sandeep Neema","raw_affiliation_strings":["Vanderbilt University,Electrical and Computer Engineering","Electrical and Computer Engineering, Vanderbilt University"],"affiliations":[{"raw_affiliation_string":"Vanderbilt University,Electrical and Computer Engineering","institution_ids":["https://openalex.org/I200719446"]},{"raw_affiliation_string":"Electrical and Computer Engineering, Vanderbilt University","institution_ids":["https://openalex.org/I200719446"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":7,"corresponding_author_ids":["https://openalex.org/A5075978538"],"corresponding_institution_ids":["https://openalex.org/I1298353152","https://openalex.org/I19700959"],"apc_list":null,"apc_paid":null,"fwci":4.5889,"has_fulltext":false,"cited_by_count":10,"citation_normalized_percentile":{"value":0.95320206,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":91,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"944","last_page":"949"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9957000017166138,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9957000017166138,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9951000213623047,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9922999739646912,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.747187614440918},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.5910890102386475},{"id":"https://openalex.org/keywords/natural-language","display_name":"Natural language","score":0.5768746733665466},{"id":"https://openalex.org/keywords/semantic-reasoner","display_name":"Semantic reasoner","score":0.5142485499382019},{"id":"https://openalex.org/keywords/dialog-box","display_name":"Dialog box","score":0.4652261435985565},{"id":"https://openalex.org/keywords/domain-specific-language","display_name":"Domain-specific language","score":0.42701026797294617},{"id":"https://openalex.org/keywords/counterexample","display_name":"Counterexample","score":0.4131360650062561},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3960344195365906},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.3303176164627075}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.747187614440918},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.5910890102386475},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.5768746733665466},{"id":"https://openalex.org/C9616225","wikidata":"https://www.wikidata.org/wiki/Q3929429","display_name":"Semantic reasoner","level":2,"score":0.5142485499382019},{"id":"https://openalex.org/C173853756","wikidata":"https://www.wikidata.org/wiki/Q86915","display_name":"Dialog box","level":2,"score":0.4652261435985565},{"id":"https://openalex.org/C135257023","wikidata":"https://www.wikidata.org/wiki/Q691358","display_name":"Domain-specific language","level":2,"score":0.42701026797294617},{"id":"https://openalex.org/C162838799","wikidata":"https://www.wikidata.org/wiki/Q596077","display_name":"Counterexample","level":2,"score":0.4131360650062561},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3960344195365906},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3303176164627075},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C118615104","wikidata":"https://www.wikidata.org/wiki/Q121416","display_name":"Discrete mathematics","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/milcom58377.2023.10356332","is_oa":false,"landing_page_url":"https://doi.org/10.1109/milcom58377.2023.10356332","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"MILCOM 2023 - 2023 IEEE Military Communications Conference (MILCOM)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320332447","display_name":"U.S. Army","ror":"https://ror.org/00afsp483"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":34,"referenced_works":["https://openalex.org/W258469679","https://openalex.org/W258782345","https://openalex.org/W1480909796","https://openalex.org/W2054497239","https://openalex.org/W2134734244","https://openalex.org/W2759136286","https://openalex.org/W2975185270","https://openalex.org/W2997049506","https://openalex.org/W3005742798","https://openalex.org/W3177813494","https://openalex.org/W4206850841","https://openalex.org/W4226485558","https://openalex.org/W4252861488","https://openalex.org/W4285428875","https://openalex.org/W4287888135","https://openalex.org/W4309674289","https://openalex.org/W4313384047","https://openalex.org/W4321011818","https://openalex.org/W4366850543","https://openalex.org/W4384302803","https://openalex.org/W4385573010","https://openalex.org/W4385730880","https://openalex.org/W4387224492","https://openalex.org/W4390874280","https://openalex.org/W6768586863","https://openalex.org/W6773631536","https://openalex.org/W6798182279","https://openalex.org/W6809509765","https://openalex.org/W6810874553","https://openalex.org/W6839928859","https://openalex.org/W6847873901","https://openalex.org/W6850348513","https://openalex.org/W6851884503","https://openalex.org/W6857582342"],"related_works":["https://openalex.org/W2349952486","https://openalex.org/W1489260687","https://openalex.org/W3204670144","https://openalex.org/W2163101548","https://openalex.org/W1988934358","https://openalex.org/W2587222766","https://openalex.org/W4389574038","https://openalex.org/W4236658683","https://openalex.org/W2020463334","https://openalex.org/W2026182709"],"abstract_inverted_index":{"Generative":[0],"large":[1],"language":[2,17],"models":[3,43,75],"(LLMs)":[4],"can":[5,103,110,159,238,257],"follow":[6],"human-provided":[7],"instruction":[8],"prompts":[9],"and":[10,34,90,96,137,155,196,201,249,254],"generate":[11,165],"human-like":[12],"responses.":[13],"Apart":[14],"from":[15,127,231],"natural":[16,218,232,247],"responses,":[18],"they":[19],"have":[20],"been":[21],"found":[22],"to":[23,47,64,72,76,122,141,164,212,225,242],"be":[24,104],"effective":[25],"at":[26],"generating":[27],"formal":[28,78,98],"artifacts":[29,79,99],"such":[30,86],"as":[31,65,87,118,180,198,203],"code,":[32,94],"plans,":[33],"logical":[35],"specifications.":[36],"Despite":[37],"their":[38,56,244],"remarkably":[39],"improved":[40],"accuracy,":[41],"these":[42,74],"are":[44,135],"still":[45],"known":[46],"produce":[48,130,258],"factually":[49],"incorrect":[50],"or":[51],"contextually":[52],"inappropriate":[53],"results":[54],"despite":[55],"syntactic":[57],"coherence":[58],"\u2013":[59],"a":[60],"phenomenon":[61],"often":[62],"referred":[63],"hallucinations.":[66],"This":[67,150],"limitation":[68],"makes":[69],"it":[70],"difficult":[71],"use":[73,111,173,189],"synthesize":[77],"used":[80],"in":[81,93,217,246],"safety-critical":[82],"applications.":[83],"Unlike":[84],"tasks":[85],"text":[88],"summarization":[89],"question-answering,":[91],"bugs":[92],"plan,":[95],"other":[97],"produced":[100],"by":[101],"LLMs":[102,143,154,200,253],"catastrophic.":[105],"We":[106,188],"posit":[107],"that":[108,139],"we":[109,172],"the":[112,124,128,133,142,145,162,166,176,199,204,210,214,221,235,250],"satisfiability":[113],"modulo":[114],"theory":[115],"(SMT)":[116],"solvers":[117,158,227,256],"deductive":[119,156],"reasoning":[120],"engines":[121],"analyze":[123],"generated":[125,230],"solutions":[126,134],"LLMs,":[129],"counterexamples":[131],"when":[132],"incorrect,":[136],"provide":[138],"feedback":[140],"exploiting":[144],"dialog":[146],"capability":[147],"of":[148,178,223,252],"LLMs.":[149],"interaction":[151],"between":[152],"inductive":[153],"SMT":[157,205,226,255],"iteratively":[160],"steer":[161],"LLM":[163],"correct":[167,260],"response.":[168],"In":[169],"our":[170,181,186],"experiments,":[171],"planning":[174,215],"over":[175],"domain":[177],"blocks":[179],"synthesis":[182],"task":[183],"for":[184],"evaluating":[185],"approach.":[187],"GPT-4,":[190],"GPT3.5":[191],"Turbo,":[192],"Davinci,":[193],"Curie,":[194],"Babbage,":[195],"Ada":[197],"Z3":[202],"solver.":[206],"Our":[207],"method":[208],"allows":[209],"user":[211],"communicate":[213],"problem":[216],"language;":[219],"even":[220],"formulation":[222],"queries":[224],"is":[228],"automatically":[229],"language.":[233],"Thus,":[234],"proposed":[236],"technique":[237],"enable":[239],"non-expert":[240],"users":[241],"describe":[243],"problems":[245],"language,":[248],"combination":[251],"provably":[259],"solutions.":[261]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":6}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}