{"id":"https://openalex.org/W4411522888","doi":"https://doi.org/10.1145/3728902","title":"ConTested: Consistency-Aided Tested Code Generation with LLM","display_name":"ConTested: Consistency-Aided Tested Code Generation with LLM","publication_year":2025,"publication_date":"2025-06-22","ids":{"openalex":"https://openalex.org/W4411522888","doi":"https://doi.org/10.1145/3728902"},"language":"en","primary_location":{"id":"doi:10.1145/3728902","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3728902","pdf_url":null,"source":{"id":"https://openalex.org/S4404663975","display_name":"Proceedings of the ACM on software engineering.","issn_l":"2994-970X","issn":["2994-970X"],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Software Engineering","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://ink.library.smu.edu.sg/sis_research/10284","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5051353261","display_name":"Jinhao Dong","orcid":"https://orcid.org/0009-0009-0416-6896"},"institutions":[{"id":"https://openalex.org/I165932596","display_name":"National University of Singapore","ror":"https://ror.org/01tgyzw49","country_code":"SG","type":"education","lineage":["https://openalex.org/I165932596"]},{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]}],"countries":["CN","SG"],"is_corresponding":true,"raw_author_name":"Jinhao Dong","raw_affiliation_strings":["National University of Singapore, Singapore, Singapore","Peking University, Beijing, China"],"affiliations":[{"raw_affiliation_string":"National University of Singapore, Singapore, Singapore","institution_ids":["https://openalex.org/I165932596"]},{"raw_affiliation_string":"Peking University, Beijing, China","institution_ids":["https://openalex.org/I20231570"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100429004","display_name":"Jun Sun","orcid":"https://orcid.org/0000-0002-3545-1392"},"institutions":[{"id":"https://openalex.org/I79891267","display_name":"Singapore Management University","ror":"https://ror.org/050qmg959","country_code":"SG","type":"education","lineage":["https://openalex.org/I79891267"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Jun Sun","raw_affiliation_strings":["Singapore Management University, Singapore, Singapore"],"affiliations":[{"raw_affiliation_string":"Singapore Management University, Singapore, Singapore","institution_ids":["https://openalex.org/I79891267"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100385507","display_name":"Wenjie Zhang","orcid":"https://orcid.org/0000-0002-2669-1837"},"institutions":[{"id":"https://openalex.org/I165932596","display_name":"National University of Singapore","ror":"https://ror.org/01tgyzw49","country_code":"SG","type":"education","lineage":["https://openalex.org/I165932596"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Wenjie Zhang","raw_affiliation_strings":["National University of Singapore, Singapore, Singapore"],"affiliations":[{"raw_affiliation_string":"National University of Singapore, Singapore, Singapore","institution_ids":["https://openalex.org/I165932596"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5085067496","display_name":"Jin Song Dong","orcid":"https://orcid.org/0000-0002-6512-8326"},"institutions":[{"id":"https://openalex.org/I165932596","display_name":"National University of Singapore","ror":"https://ror.org/01tgyzw49","country_code":"SG","type":"education","lineage":["https://openalex.org/I165932596"]},{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]}],"countries":["CN","SG"],"is_corresponding":false,"raw_author_name":"Jin Song Dong","raw_affiliation_strings":["National University of Singapore, Singapore, Singapore","Peking University, Beijing, China"],"affiliations":[{"raw_affiliation_string":"National University of Singapore, Singapore, Singapore","institution_ids":["https://openalex.org/I165932596"]},{"raw_affiliation_string":"Peking University, Beijing, China","institution_ids":["https://openalex.org/I20231570"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5085393851","display_name":"Dan Hao","orcid":"https://orcid.org/0000-0001-8295-303X"},"institutions":[{"id":"https://openalex.org/I4210128628","display_name":"Peking University Shenzhen Hospital","ror":"https://ror.org/03kkjyb15","country_code":"CN","type":"healthcare","lineage":["https://openalex.org/I4210128628"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Dan Hao","raw_affiliation_strings":["Peking University, Shenzhen, China"],"affiliations":[{"raw_affiliation_string":"Peking University, Shenzhen, China","institution_ids":["https://openalex.org/I4210128628"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5051353261"],"corresponding_institution_ids":["https://openalex.org/I165932596","https://openalex.org/I20231570"],"apc_list":null,"apc_paid":null,"fwci":15.3348,"has_fulltext":false,"cited_by_count":5,"citation_normalized_percentile":{"value":0.98702032,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":91,"max":100},"biblio":{"volume":"2","issue":"ISSTA","first_page":"596","last_page":"617"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9976999759674072,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9976999759674072,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9969000220298767,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.992900013923645,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/consistency","display_name":"Consistency (knowledge bases)","score":0.8661490678787231},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6975652575492859},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.6762362122535706},{"id":"https://openalex.org/keywords/debugging","display_name":"Debugging","score":0.6208631992340088},{"id":"https://openalex.org/keywords/overhead","display_name":"Overhead (engineering)","score":0.5955313444137573},{"id":"https://openalex.org/keywords/voting","display_name":"Voting","score":0.5834180116653442},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.5201035737991333},{"id":"https://openalex.org/keywords/rank","display_name":"Rank (graph theory)","score":0.5115290880203247},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.2032138705253601},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.20210787653923035},{"id":"https://openalex.org/keywords/political-science","display_name":"Political science","score":0.06560057401657104}],"concepts":[{"id":"https://openalex.org/C2776436953","wikidata":"https://www.wikidata.org/wiki/Q5163215","display_name":"Consistency (knowledge bases)","level":2,"score":0.8661490678787231},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6975652575492859},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.6762362122535706},{"id":"https://openalex.org/C168065819","wikidata":"https://www.wikidata.org/wiki/Q845566","display_name":"Debugging","level":2,"score":0.6208631992340088},{"id":"https://openalex.org/C2779960059","wikidata":"https://www.wikidata.org/wiki/Q7113681","display_name":"Overhead (engineering)","level":2,"score":0.5955313444137573},{"id":"https://openalex.org/C520049643","wikidata":"https://www.wikidata.org/wiki/Q189760","display_name":"Voting","level":3,"score":0.5834180116653442},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.5201035737991333},{"id":"https://openalex.org/C164226766","wikidata":"https://www.wikidata.org/wiki/Q7293202","display_name":"Rank (graph theory)","level":2,"score":0.5115290880203247},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2032138705253601},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.20210787653923035},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.06560057401657104},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C94625758","wikidata":"https://www.wikidata.org/wiki/Q7163","display_name":"Politics","level":2,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.0},{"id":"https://openalex.org/C114614502","wikidata":"https://www.wikidata.org/wiki/Q76592","display_name":"Combinatorics","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3728902","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3728902","pdf_url":null,"source":{"id":"https://openalex.org/S4404663975","display_name":"Proceedings of the ACM on software engineering.","issn_l":"2994-970X","issn":["2994-970X"],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Software Engineering","raw_type":"journal-article"},{"id":"pmh:oai:ink.library.smu.edu.sg:sis_research-11285","is_oa":true,"landing_page_url":"https://ink.library.smu.edu.sg/sis_research/10284","pdf_url":null,"source":{"id":"https://openalex.org/S4306401925","display_name":"Singapore Management University Institutional Knowledge (InK) (Singapore Management University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I79891267","host_organization_name":"Singapore Management University","host_organization_lineage":["https://openalex.org/I79891267"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"https://doi.org/10.1145/3728902","raw_type":"Conference Proceeding Article"}],"best_oa_location":{"id":"pmh:oai:ink.library.smu.edu.sg:sis_research-11285","is_oa":true,"landing_page_url":"https://ink.library.smu.edu.sg/sis_research/10284","pdf_url":null,"source":{"id":"https://openalex.org/S4306401925","display_name":"Singapore Management University Institutional Knowledge (InK) (Singapore Management University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I79891267","host_organization_name":"Singapore Management University","host_organization_lineage":["https://openalex.org/I79891267"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"https://doi.org/10.1145/3728902","raw_type":"Conference Proceeding Article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":12,"referenced_works":["https://openalex.org/W2618735189","https://openalex.org/W2914304175","https://openalex.org/W2962728167","https://openalex.org/W2997847174","https://openalex.org/W4377238789","https://openalex.org/W4377866025","https://openalex.org/W4384345748","https://openalex.org/W4385572707","https://openalex.org/W4389519889","https://openalex.org/W4399557965","https://openalex.org/W4402671965","https://openalex.org/W4402683938"],"related_works":["https://openalex.org/W4321442002","https://openalex.org/W2015265939","https://openalex.org/W2284072287","https://openalex.org/W2611067230","https://openalex.org/W2480201319","https://openalex.org/W2387706296","https://openalex.org/W2155788121","https://openalex.org/W4235469518","https://openalex.org/W362492756","https://openalex.org/W2890345561"],"abstract_inverted_index":{"Recent":[0],"advancements":[1],"in":[2],"large":[3],"language":[4,20],"models":[5],"(LLMs)":[6],"have":[7,48],"significantly":[8,118],"improved":[9],"code":[10,14,54,130,146,222],"generation,":[11],"which":[12],"generates":[13],"snippets":[15],"automatically":[16],"based":[17,82],"on":[18,79,83],"natural":[19],"requirements.":[21],"Despite":[22],"achieving":[23],"state-of-the-art":[24],"performance,":[25],"LLMs":[26],"often":[27],"struggle":[28],"to":[29,37,52,87,103,220],"generate":[30],"accurate":[31],"and":[32,42,60,131,147,165,175,213],"reliable":[33],"code,":[34],"requiring":[35,202],"developers":[36],"spend":[38],"substantial":[39,226],"effort":[40],"debugging":[41],"evaluating":[43],"the":[44,70,137,142,186,211],"generated":[45,75],"output.":[46],"Researchers":[47],"proposed":[49],"leveraging":[50],"Consistency":[51],"select":[53],"that":[55,99],"passes":[56],"more":[57,65,149],"tests":[58,71,85,148],"(inter-consistency)":[59],"demonstrates":[61],"consistent":[62],"behavior":[63],"across":[64,160],"counterparts":[66],"(intra-consistency).":[67],"However,":[68],"since":[69],"themselves":[72],"are":[73],"also":[74],"by":[76],"LLMs,":[77,162],"relying":[78],"majority":[80],"voting":[81,144],"incorrect":[84],"leads":[86],"unreliable":[88],"results.":[89],"To":[90],"address":[91],"this,":[92],"we":[93,123],"propose":[94],"a":[95,125,197],"lightweight":[96],"interaction":[97,199],"framework":[98],"incorporates":[100],"user":[101,204,207],"feedback":[102],"effectively":[104],"guide":[105],"consistency.":[106],"Our":[107,167],"results":[108,168],"demonstrate":[109],"that,":[110],"with":[111,195,200],"minimal":[112,203],"human":[113],"effort,":[114],"performance":[115],"can":[116],"be":[117],"improved.":[119],"In":[120],"each":[121],"iteration,":[122],"introduce":[124],"rank-correct-fix":[126],"co-evolution":[127],"process":[128,134],"between":[129,145],"tests.":[132],"This":[133,191],"iteratively":[135],"enhances":[136],"quality":[138],"of":[139,171,215],"both,":[140],"making":[141],"consistency":[143],"reliable.":[150],"We":[151],"evaluate":[152],"ConTested":[153,180],"through":[154],"extensive":[155],"experiments,":[156],"demonstrating":[157],"its":[158,218],"effectiveness":[159],"multiple":[161],"including":[163],"GPT-3.5":[164,174],"GPT-4o.":[166,178],"show":[169],"improvements":[170],"32.9%":[172],"over":[173,177,185],"16.97%":[176],"Additionally,":[179],"achieves":[181],"an":[182],"11.1%":[183],"improvement":[184,192],"SOTA":[187],"post-processing":[188],"technique,":[189],"MPSC.":[190],"is":[193],"achieved":[194],"only":[196],"4-round":[198],"users,":[201],"effort.":[205],"A":[206],"study":[208],"further":[209],"confirms":[210],"feasibility":[212],"cost-effectiveness":[214],"ConTested,":[216],"highlighting":[217],"ability":[219],"enhance":[221],"generation":[223],"without":[224],"introducing":[225],"overhead.":[227]},"counts_by_year":[{"year":2026,"cited_by_count":4},{"year":2025,"cited_by_count":1}],"updated_date":"2026-04-11T08:14:18.477133","created_date":"2025-10-10T00:00:00"}