{"id":"https://openalex.org/W4389966777","doi":"https://doi.org/10.1145/3627217.3627238","title":"Creating Thorough Tests for AI-Generated Code is Hard","display_name":"Creating Thorough Tests for AI-Generated Code is Hard","publication_year":2023,"publication_date":"2023-12-09","ids":{"openalex":"https://openalex.org/W4389966777","doi":"https://doi.org/10.1145/3627217.3627238"},"language":"en","primary_location":{"id":"doi:10.1145/3627217.3627238","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3627217.3627238","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 16th Annual ACM India Compute Conference","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5025197204","display_name":"Shreya Singhal","orcid":null},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Shreya Singhal","raw_affiliation_strings":["Indian Institute of Technology, India"],"affiliations":[{"raw_affiliation_string":"Indian Institute of Technology, India","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5090940680","display_name":"Viraj Kumar","orcid":"https://orcid.org/0000-0002-2252-0141"},"institutions":[{"id":"https://openalex.org/I59270414","display_name":"Indian Institute of Science Bangalore","ror":"https://ror.org/04dese585","country_code":"IN","type":"education","lineage":["https://openalex.org/I59270414"]}],"countries":["IN"],"is_corresponding":false,"raw_author_name":"Viraj Kumar","raw_affiliation_strings":["Indian Institute of Science, India"],"affiliations":[{"raw_affiliation_string":"Indian Institute of Science, India","institution_ids":["https://openalex.org/I59270414"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":["https://openalex.org/A5025197204"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.4311,"has_fulltext":false,"cited_by_count":2,"citation_normalized_percentile":{"value":0.7141626,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":95,"max":96},"biblio":{"volume":null,"issue":null,"first_page":"108","last_page":"111"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12423","display_name":"Software Reliability and Analysis Research","score":0.9950000047683716,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/test-suite","display_name":"Test suite","score":0.8909468650817871},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8453217148780823},{"id":"https://openalex.org/keywords/suite","display_name":"Suite","score":0.7527912259101868},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.7056184411048889},{"id":"https://openalex.org/keywords/test","display_name":"Test (biology)","score":0.6907002925872803},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.622911274433136},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.48008328676223755},{"id":"https://openalex.org/keywords/code-coverage","display_name":"Code coverage","score":0.4571416974067688},{"id":"https://openalex.org/keywords/function","display_name":"Function (biology)","score":0.42215263843536377},{"id":"https://openalex.org/keywords/test-case","display_name":"Test case","score":0.4017086625099182},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.21153688430786133},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.19450625777244568}],"concepts":[{"id":"https://openalex.org/C151552104","wikidata":"https://www.wikidata.org/wiki/Q7705809","display_name":"Test suite","level":4,"score":0.8909468650817871},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8453217148780823},{"id":"https://openalex.org/C79581498","wikidata":"https://www.wikidata.org/wiki/Q1367530","display_name":"Suite","level":2,"score":0.7527912259101868},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.7056184411048889},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.6907002925872803},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.622911274433136},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.48008328676223755},{"id":"https://openalex.org/C53942775","wikidata":"https://www.wikidata.org/wiki/Q1211721","display_name":"Code coverage","level":3,"score":0.4571416974067688},{"id":"https://openalex.org/C14036430","wikidata":"https://www.wikidata.org/wiki/Q3736076","display_name":"Function (biology)","level":2,"score":0.42215263843536377},{"id":"https://openalex.org/C128942645","wikidata":"https://www.wikidata.org/wiki/Q1568346","display_name":"Test case","level":3,"score":0.4017086625099182},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.21153688430786133},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.19450625777244568},{"id":"https://openalex.org/C78458016","wikidata":"https://www.wikidata.org/wiki/Q840400","display_name":"Evolutionary biology","level":1,"score":0.0},{"id":"https://openalex.org/C95457728","wikidata":"https://www.wikidata.org/wiki/Q309","display_name":"History","level":0,"score":0.0},{"id":"https://openalex.org/C152877465","wikidata":"https://www.wikidata.org/wiki/Q208042","display_name":"Regression analysis","level":2,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C166957645","wikidata":"https://www.wikidata.org/wiki/Q23498","display_name":"Archaeology","level":1,"score":0.0},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3627217.3627238","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3627217.3627238","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 16th Annual ACM India Compute Conference","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4","score":0.6899999976158142}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":13,"referenced_works":["https://openalex.org/W1521965716","https://openalex.org/W1992987499","https://openalex.org/W2121077396","https://openalex.org/W2129761255","https://openalex.org/W2786944871","https://openalex.org/W2965029801","https://openalex.org/W2966980041","https://openalex.org/W2990912491","https://openalex.org/W3093870681","https://openalex.org/W3124323960","https://openalex.org/W3194305021","https://openalex.org/W4323037544","https://openalex.org/W4324138978"],"related_works":["https://openalex.org/W4295918990","https://openalex.org/W4313447549","https://openalex.org/W2028796071","https://openalex.org/W2067499928","https://openalex.org/W1888619389","https://openalex.org/W2991649730","https://openalex.org/W3022870375","https://openalex.org/W2794522096","https://openalex.org/W1968494916","https://openalex.org/W2052414005"],"abstract_inverted_index":{"Before":[0],"implementing":[1],"a":[2,9,101],"function,":[3],"programmers":[4],"are":[5,65],"encouraged":[6],"to":[7,60,68,88,122],"write":[8],"suite":[10,23],"of":[11,24,36,45,58,120],"test":[12,62,98,111,125],"cases":[13],"that":[14,41,64],"specify":[15],"its":[16],"intended":[17],"behaviour":[18],"on":[19],"several":[20],"inputs.":[21],"A":[22],"tests":[25,91],"is":[26],"thorough":[27,66,90,124],"if":[28],"any":[29],"buggy":[30],"implementation":[31],"fails":[32],"at":[33],"least":[34],"one":[35],"these":[37,110],"tests.":[38],"We":[39],"posit":[40],"as":[42],"the":[43,56,118],"proportion":[44],"code":[46,94],"generated":[47],"by":[48,95,106],"Large":[49],"Language":[50],"Models":[51],"(LLMs)":[52],"grows,":[53],"so":[54],"must":[55],"ability":[57,119],"students":[59,121],"create":[61,89],"suites":[63,99,126],"enough":[67],"detect":[69],"subtle":[70],"bugs":[71],"in":[72,109],"such":[73],"code.":[74,129],"Our":[75],"paper":[76],"makes":[77],"two":[78],"contributions.":[79],"First,":[80],"we":[81,113],"demonstrate":[82],"how":[83],"difficult":[84],"it":[85],"can":[86],"be":[87],"for":[92,116,127],"LLM-generated":[93,128],"evaluating":[96],"27":[97],"from":[100],"public":[102],"dataset":[103],"(EvalPlus).":[104],"Second,":[105],"identifying":[107],"deficiencies":[108],"suites,":[112],"propose":[114],"strategies":[115],"improving":[117],"develop":[123]},"counts_by_year":[{"year":2025,"cited_by_count":2}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}