{"id":"https://openalex.org/W4398239202","doi":"https://doi.org/10.1145/3639478.3643525","title":"Exploring the Effectiveness of LLM based Test-driven Interactive Code Generation: User Study and Empirical Evaluation","display_name":"Exploring the Effectiveness of LLM based Test-driven Interactive Code Generation: User Study and Empirical Evaluation","publication_year":2024,"publication_date":"2024-04-14","ids":{"openalex":"https://openalex.org/W4398239202","doi":"https://doi.org/10.1145/3639478.3643525"},"language":"en","primary_location":{"id":"doi:10.1145/3639478.3643525","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3639478.3643525","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3639478.3643525","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2024 IEEE/ACM 46th International Conference on Software Engineering: Companion Proceedings","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3639478.3643525","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5068417203","display_name":"Sarah Fakhoury","orcid":"https://orcid.org/0000-0002-8486-7749"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Sarah Fakhoury","raw_affiliation_strings":["Microsoft Research, Redmond, Washington, USA"],"affiliations":[{"raw_affiliation_string":"Microsoft Research, Redmond, Washington, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5060397379","display_name":"Aaditya Naik","orcid":"https://orcid.org/0000-0002-3100-0455"},"institutions":[{"id":"https://openalex.org/I79576946","display_name":"University of Pennsylvania","ror":"https://ror.org/00b30xv10","country_code":"US","type":"education","lineage":["https://openalex.org/I79576946"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Aaditya Naik","raw_affiliation_strings":["University of Pennsylvania, Philadelphia, Pennsylvania, USA"],"affiliations":[{"raw_affiliation_string":"University of Pennsylvania, Philadelphia, Pennsylvania, USA","institution_ids":["https://openalex.org/I79576946"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5007730160","display_name":"Georgios K. Sakkas","orcid":"https://orcid.org/0000-0002-1071-8038"},"institutions":[{"id":"https://openalex.org/I2800935791","display_name":"UC San Diego Health System","ror":"https://ror.org/01kbfgm16","country_code":"US","type":"healthcare","lineage":["https://openalex.org/I2800935791"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Georgios Sakkas","raw_affiliation_strings":["UC San Diego, San Diego, California, USA"],"affiliations":[{"raw_affiliation_string":"UC San Diego, San Diego, California, USA","institution_ids":["https://openalex.org/I2800935791"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5074368173","display_name":"Saikat Chakraborty","orcid":"https://orcid.org/0000-0002-6889-7171"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Saikat Chakraborty","raw_affiliation_strings":["Microsoft Research, Redmond, Washington, USA"],"affiliations":[{"raw_affiliation_string":"Microsoft Research, Redmond, Washington, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5108772759","display_name":"Madan Musuvathi","orcid":null},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Madan Musuvathi","raw_affiliation_strings":["Microsoft Research, Redmond, Washington, USA"],"affiliations":[{"raw_affiliation_string":"Microsoft Research, Redmond, Washington, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5041084431","display_name":"Shuvendu K. Lahiri","orcid":"https://orcid.org/0000-0002-4446-4777"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Shuvendu Lahiri","raw_affiliation_strings":["Microsoft Research, Redmond, Washington, USA"],"affiliations":[{"raw_affiliation_string":"Microsoft Research, Redmond, Washington, USA","institution_ids":["https://openalex.org/I1290206253"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5068417203"],"corresponding_institution_ids":["https://openalex.org/I1290206253"],"apc_list":null,"apc_paid":null,"fwci":6.5656,"has_fulltext":false,"cited_by_count":8,"citation_normalized_percentile":{"value":0.96567884,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"390","last_page":"391"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.995199978351593,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11241","display_name":"Advanced Malware Detection Techniques","score":0.9797999858856201,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8152695298194885},{"id":"https://openalex.org/keywords/workflow","display_name":"Workflow","score":0.6835774779319763},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.6142362952232361},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.5624353885650635},{"id":"https://openalex.org/keywords/empirical-research","display_name":"Empirical research","score":0.5462120175361633},{"id":"https://openalex.org/keywords/test","display_name":"Test (biology)","score":0.5235395431518555},{"id":"https://openalex.org/keywords/code-generation","display_name":"Code generation","score":0.486870676279068},{"id":"https://openalex.org/keywords/natural-language-generation","display_name":"Natural language generation","score":0.41820722818374634},{"id":"https://openalex.org/keywords/natural-language","display_name":"Natural language","score":0.3689495325088501},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.3532603979110718},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.3476184606552124},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.32321590185165405},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.28400474786758423},{"id":"https://openalex.org/keywords/computer-security","display_name":"Computer security","score":0.12445226311683655},{"id":"https://openalex.org/keywords/database","display_name":"Database","score":0.11441755294799805}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8152695298194885},{"id":"https://openalex.org/C177212765","wikidata":"https://www.wikidata.org/wiki/Q627335","display_name":"Workflow","level":2,"score":0.6835774779319763},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.6142362952232361},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.5624353885650635},{"id":"https://openalex.org/C120936955","wikidata":"https://www.wikidata.org/wiki/Q2155640","display_name":"Empirical research","level":2,"score":0.5462120175361633},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.5235395431518555},{"id":"https://openalex.org/C133162039","wikidata":"https://www.wikidata.org/wiki/Q1061077","display_name":"Code generation","level":3,"score":0.486870676279068},{"id":"https://openalex.org/C2776187449","wikidata":"https://www.wikidata.org/wiki/Q1513879","display_name":"Natural language generation","level":3,"score":0.41820722818374634},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.3689495325088501},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.3532603979110718},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.3476184606552124},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.32321590185165405},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.28400474786758423},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.12445226311683655},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.11441755294799805},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3639478.3643525","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3639478.3643525","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3639478.3643525","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2024 IEEE/ACM 46th International Conference on Software Engineering: Companion Proceedings","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3639478.3643525","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3639478.3643525","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3639478.3643525","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2024 IEEE/ACM 46th International Conference on Software Engineering: Companion Proceedings","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":false},"content_urls":{"pdf":"https://content.openalex.org/works/W4398239202.pdf"},"referenced_works_count":3,"referenced_works":["https://openalex.org/W258469679","https://openalex.org/W3177813494","https://openalex.org/W4287024925"],"related_works":["https://openalex.org/W2152921782","https://openalex.org/W382594479","https://openalex.org/W2470045054","https://openalex.org/W2575772232","https://openalex.org/W2151245229","https://openalex.org/W2140902089","https://openalex.org/W2030298461","https://openalex.org/W1510553545","https://openalex.org/W3020827637","https://openalex.org/W199086061"],"abstract_inverted_index":{"We":[0,47],"introduce":[1],"a":[2,34,76],"novel":[3],"workflow,":[4],"TiCoder,":[5],"designed":[6],"to":[7,38],"enhance":[8],"the":[9,49,65],"trust":[10],"and":[11,19,63],"accuracy":[12,52],"of":[13,36,69],"LLM-based":[14],"code":[15,45,50],"generation":[16,51],"through":[17,75],"interactive":[18],"guided":[20],"intent":[21,27],"formalization.":[22],"TiCoder":[23,56,74],"partially":[24],"formalizes":[25],"ambiguous":[26],"in":[28,43],"natural":[29],"language":[30],"prompts":[31],"by":[32,55,73],"generating":[33],"set":[35],"tests":[37,71],"distinguish":[39],"common":[40],"divergent":[41],"behaviours":[42],"generated":[44],"suggestions.":[46],"evaluate":[48,64],"improvements":[53],"provided":[54],"at":[57],"scale":[58],"across":[59],"four":[60],"competitive":[61],"LLMs,":[62],"cost-benefit":[66],"trade":[67],"off":[68],"evaluating":[70],"surfaced":[72],"user":[77],"study":[78],"with":[79],"15":[80],"participants.":[81]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":5},{"year":2024,"cited_by_count":1}],"updated_date":"2026-03-04T09:10:02.777135","created_date":"2025-10-10T00:00:00"}