{"id":"https://openalex.org/W4403186335","doi":"https://doi.org/10.1145/3699598","title":"Exploring Automated Assertion Generation via Large Language Models","display_name":"Exploring Automated Assertion Generation via Large Language Models","publication_year":2024,"publication_date":"2024-10-07","ids":{"openalex":"https://openalex.org/W4403186335","doi":"https://doi.org/10.1145/3699598"},"language":"en","primary_location":{"id":"doi:10.1145/3699598","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3699598","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3699598","source":{"id":"https://openalex.org/S142627899","display_name":"ACM Transactions on Software Engineering and Methodology","issn_l":"1049-331X","issn":["1049-331X","1557-7392"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Software Engineering and Methodology","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"bronze","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3699598","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5101756397","display_name":"Quanjun Zhang","orcid":"https://orcid.org/0000-0002-2495-3805"},"institutions":[{"id":"https://openalex.org/I881766915","display_name":"Nanjing University","ror":"https://ror.org/01rxvg760","country_code":"CN","type":"education","lineage":["https://openalex.org/I881766915"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Quanjun Zhang","raw_affiliation_strings":["State Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, China","State Key Laboratory for Novel Software Technology, Nanjing University, China"],"raw_orcid":"https://orcid.org/0000-0002-2495-3805","affiliations":[{"raw_affiliation_string":"State Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, China","institution_ids":["https://openalex.org/I881766915"]},{"raw_affiliation_string":"State Key Laboratory for Novel Software Technology, Nanjing University, China","institution_ids":["https://openalex.org/I881766915"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101650869","display_name":"Weifeng Sun","orcid":"https://orcid.org/0000-0001-6013-1369"},"institutions":[{"id":"https://openalex.org/I158842170","display_name":"Chongqing University","ror":"https://ror.org/023rhb549","country_code":"CN","type":"education","lineage":["https://openalex.org/I158842170"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Weifeng Sun","raw_affiliation_strings":["School of Big Data and Software Engineering, Chongqing University, Chongqing, China","School of Big Data & Software Engineering, Chongqing University, China"],"raw_orcid":"https://orcid.org/0000-0001-6013-1369","affiliations":[{"raw_affiliation_string":"School of Big Data and Software Engineering, Chongqing University, Chongqing, China","institution_ids":["https://openalex.org/I158842170"]},{"raw_affiliation_string":"School of Big Data & Software Engineering, Chongqing University, China","institution_ids":["https://openalex.org/I158842170"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5075174750","display_name":"Chunrong Fang","orcid":"https://orcid.org/0000-0002-9930-7111"},"institutions":[{"id":"https://openalex.org/I881766915","display_name":"Nanjing University","ror":"https://ror.org/01rxvg760","country_code":"CN","type":"education","lineage":["https://openalex.org/I881766915"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Chunrong Fang","raw_affiliation_strings":["State Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, China","State Key Laboratory for Novel Software Technology, Nanjing University, China"],"raw_orcid":"https://orcid.org/0000-0002-9930-7111","affiliations":[{"raw_affiliation_string":"State Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, China","institution_ids":["https://openalex.org/I881766915"]},{"raw_affiliation_string":"State Key Laboratory for Novel Software Technology, Nanjing University, China","institution_ids":["https://openalex.org/I881766915"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101188063","display_name":"Bowen Yu","orcid":"https://orcid.org/0009-0008-9642-7880"},"institutions":[{"id":"https://openalex.org/I881766915","display_name":"Nanjing University","ror":"https://ror.org/01rxvg760","country_code":"CN","type":"education","lineage":["https://openalex.org/I881766915"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Bowen Yu","raw_affiliation_strings":["State Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, China","State Key Laboratory for Novel Software Technology, Nanjing University, China"],"raw_orcid":"https://orcid.org/0009-0008-9642-7880","affiliations":[{"raw_affiliation_string":"State Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, China","institution_ids":["https://openalex.org/I881766915"]},{"raw_affiliation_string":"State Key Laboratory for Novel Software Technology, Nanjing University, China","institution_ids":["https://openalex.org/I881766915"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5019248426","display_name":"Hongyan Li","orcid":"https://orcid.org/0000-0003-2204-4648"},"institutions":[{"id":"https://openalex.org/I158842170","display_name":"Chongqing University","ror":"https://ror.org/023rhb549","country_code":"CN","type":"education","lineage":["https://openalex.org/I158842170"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Hongyan Li","raw_affiliation_strings":["School of Big Data and Software Engineering, Chongqing University, Chongqing, China","School of Big Data & Software Engineering, Chongqing University, China"],"raw_orcid":"https://orcid.org/0000-0003-2204-4648","affiliations":[{"raw_affiliation_string":"School of Big Data and Software Engineering, Chongqing University, Chongqing, China","institution_ids":["https://openalex.org/I158842170"]},{"raw_affiliation_string":"School of Big Data & Software Engineering, Chongqing University, China","institution_ids":["https://openalex.org/I158842170"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5070783668","display_name":"Meng Yan","orcid":"https://orcid.org/0000-0002-9538-9121"},"institutions":[{"id":"https://openalex.org/I158842170","display_name":"Chongqing University","ror":"https://ror.org/023rhb549","country_code":"CN","type":"education","lineage":["https://openalex.org/I158842170"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Meng Yan","raw_affiliation_strings":["School of Big Data and Software Engineering, Chongqing University, Chongqing, China","School of Big Data & Software Engineering, Chongqing University, China"],"raw_orcid":"https://orcid.org/0000-0002-9538-9121","affiliations":[{"raw_affiliation_string":"School of Big Data and Software Engineering, Chongqing University, Chongqing, China","institution_ids":["https://openalex.org/I158842170"]},{"raw_affiliation_string":"School of Big Data & Software Engineering, Chongqing University, China","institution_ids":["https://openalex.org/I158842170"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101670110","display_name":"Jianyi Zhou","orcid":"https://orcid.org/0000-0002-4867-5416"},"institutions":[{"id":"https://openalex.org/I2250955327","display_name":"Huawei Technologies (China)","ror":"https://ror.org/00cmhce21","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250955327"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jianyi Zhou","raw_affiliation_strings":["Huawei Cloud Computing Technologies Co., Ltd., Beijing, China","Huawei Cloud Computing Technologies Co., Ltd., China"],"raw_orcid":"https://orcid.org/0000-0002-4867-5416","affiliations":[{"raw_affiliation_string":"Huawei Cloud Computing Technologies Co., Ltd., Beijing, China","institution_ids":["https://openalex.org/I2250955327"]},{"raw_affiliation_string":"Huawei Cloud Computing Technologies Co., Ltd., China","institution_ids":["https://openalex.org/I2250955327"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100422935","display_name":"Zhenyu Chen","orcid":"https://orcid.org/0000-0002-9592-7022"},"institutions":[{"id":"https://openalex.org/I881766915","display_name":"Nanjing University","ror":"https://ror.org/01rxvg760","country_code":"CN","type":"education","lineage":["https://openalex.org/I881766915"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhenyu Chen","raw_affiliation_strings":["State Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, China","State Key Laboratory for Novel Software Technology, Nanjing University, China"],"raw_orcid":"https://orcid.org/0000-0002-9592-7022","affiliations":[{"raw_affiliation_string":"State Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, China","institution_ids":["https://openalex.org/I881766915"]},{"raw_affiliation_string":"State Key Laboratory for Novel Software Technology, Nanjing University, China","institution_ids":["https://openalex.org/I881766915"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":5.9451,"has_fulltext":false,"cited_by_count":16,"citation_normalized_percentile":{"value":0.96957404,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":99,"max":100},"biblio":{"volume":"34","issue":"3","first_page":"1","last_page":"25"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12423","display_name":"Software Reliability and Analysis Research","score":0.9980000257492065,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/assertion","display_name":"Assertion","score":0.8818459510803223},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8141680955886841},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.46321114897727966},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.4226579964160919}],"concepts":[{"id":"https://openalex.org/C40422974","wikidata":"https://www.wikidata.org/wiki/Q741248","display_name":"Assertion","level":2,"score":0.8818459510803223},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8141680955886841},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.46321114897727966},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.4226579964160919}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3699598","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3699598","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3699598","source":{"id":"https://openalex.org/S142627899","display_name":"ACM Transactions on Software Engineering and Methodology","issn_l":"1049-331X","issn":["1049-331X","1557-7392"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Software Engineering and Methodology","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1145/3699598","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3699598","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3699598","source":{"id":"https://openalex.org/S142627899","display_name":"ACM Transactions on Software Engineering and Methodology","issn_l":"1049-331X","issn":["1049-331X","1557-7392"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Software Engineering and Methodology","raw_type":"journal-article"},"sustainable_development_goals":[{"score":0.5799999833106995,"id":"https://metadata.un.org/sdg/8","display_name":"Decent work and economic growth"}],"awards":[{"id":"https://openalex.org/G5664872752","display_name":null,"funder_award_id":"61932012, 62372228","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"pdf":true,"grobid_xml":false},"content_urls":{"pdf":"https://content.openalex.org/works/W4403186335.pdf"},"referenced_works_count":59,"referenced_works":["https://openalex.org/W1753482797","https://openalex.org/W1965194038","https://openalex.org/W1971650562","https://openalex.org/W1993760289","https://openalex.org/W2002753905","https://openalex.org/W2025791343","https://openalex.org/W2077273779","https://openalex.org/W2107709519","https://openalex.org/W2111990314","https://openalex.org/W2118655104","https://openalex.org/W2156723666","https://openalex.org/W2471601946","https://openalex.org/W2682664750","https://openalex.org/W2747329762","https://openalex.org/W2879105418","https://openalex.org/W2930957955","https://openalex.org/W2963341956","https://openalex.org/W2972082064","https://openalex.org/W2981852735","https://openalex.org/W3005628256","https://openalex.org/W3086938529","https://openalex.org/W3098605233","https://openalex.org/W3101193881","https://openalex.org/W3105903381","https://openalex.org/W3149821397","https://openalex.org/W3161997752","https://openalex.org/W3170092793","https://openalex.org/W3184041223","https://openalex.org/W3189653670","https://openalex.org/W3198685994","https://openalex.org/W3211801722","https://openalex.org/W4221166942","https://openalex.org/W4226485558","https://openalex.org/W4236219643","https://openalex.org/W4237492309","https://openalex.org/W4281479826","https://openalex.org/W4284681867","https://openalex.org/W4284690374","https://openalex.org/W4284709233","https://openalex.org/W4285204876","https://openalex.org/W4288089799","https://openalex.org/W4308641648","https://openalex.org/W4308643152","https://openalex.org/W4312436517","https://openalex.org/W4375959406","https://openalex.org/W4376122390","https://openalex.org/W4376167329","https://openalex.org/W4378942602","https://openalex.org/W4384155677","https://openalex.org/W4384302749","https://openalex.org/W4384345698","https://openalex.org/W4384345708","https://openalex.org/W4387687220","https://openalex.org/W4388422146","https://openalex.org/W4388482966","https://openalex.org/W4388502390","https://openalex.org/W4388631682","https://openalex.org/W4389104713","https://openalex.org/W4389519352"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2748952813","https://openalex.org/W1498103021","https://openalex.org/W4230849338","https://openalex.org/W4295166216","https://openalex.org/W2177044681","https://openalex.org/W1968067090","https://openalex.org/W2345942070","https://openalex.org/W2141398161","https://openalex.org/W2016410697"],"abstract_inverted_index":{"Unit":[0],"testing":[1,30,73,334],"aims":[2],"to":[3,32,55,61,109,187,251,274,317],"validate":[4],"the":[5,68,83,101,173,201,209,220,241,266,276,291,303,310,329],"correctness":[6],"of":[7,82,85,132,165,175,198,244,268,280,293,313,332],"software":[8,18],"system":[9],"units":[10],"and":[11,20,27,43,65,88,91,134,140,155,167,182,271,278,320,327],"has":[12,77],"become":[13],"an":[14,162,196],"essential":[15],"practice":[16],"in":[17,70,178,302,323,336],"development":[19],"maintenance.":[21],"However,":[22,75],"it":[23],"is":[24],"incredibly":[25],"time-consuming":[26],"labor-intensive":[28],"for":[29,229,297],"experts":[31,335],"write":[33],"unit":[34,63,333],"test":[35,39,44,272,325],"cases":[36,326],"manually,":[37],"including":[38],"inputs":[40],"(i.e.,":[41,46],"prefixes)":[42],"oracles":[45],"assertions).":[47],"Very":[48],"recently,":[49],"some":[50],"techniques":[51,127],"have":[52,66],"been":[53,78],"proposed":[54],"apply":[56],"Large":[57],"Language":[58],"Models":[59],"(LLMs)":[60],"generate":[62,318],"assertions":[64,177,322],"proven":[67],"potential":[69],"reducing":[71],"manual":[72,330],"efforts.":[74],"there":[76],"no":[79],"systematic":[80],"comparison":[81],"effectiveness":[84],"these":[86],"LLMs,":[87],"their":[89],"pros":[90],"cons":[92],"remain":[93],"unexplored.":[94],"To":[95],"bridge":[96],"this":[97],"gap,":[98],"we":[99,211,258,284],"perform":[100],"first":[102],"extensive":[103],"study":[104],"on":[105,116,142,158,193,248,254],"applying":[106],"various":[107,287],"LLMs":[108,123,154,184,245,316],"automated":[110],"assertion":[111,221,269,300],"generation.":[112],"The":[113,136],"experimental":[114],"results":[115],"two":[117],"independent":[118],"datasets":[119],"show":[120],"that":[121],"studied":[122,153],"outperform":[124],"six":[125],"state-of-the-art":[126],"with":[128,161,195],"a":[129,146,213,225,235],"prediction":[130,242],"accuracy":[131,243],"51.82%\u201358.71%":[133],"38.72%\u201348.19%.":[135],"improvements":[137],"achieve":[138],"29.60%":[139],"12.47%":[141],"average.":[143],"Besides,":[144,257],"as":[145],"representative":[147],"LLM,":[148],"CodeT5":[149],"consistently":[150],"outperforms":[151],"all":[152,156],"baselines":[157],"both":[159,255],"datasets,":[160],"average":[163],"improvement":[164,197,292],"13.85%":[166],"26.64%,":[168],"respectively.":[169],"We":[170],"also":[171],"explore":[172],"performance":[174],"generated":[176],"detecting":[179],"real-world":[180,324],"bugs,":[181],"find":[183],"are":[185],"able":[186],"detect":[188],"32":[189],"bugs":[190],"from":[191,262],"Defects4J":[192],"average,":[194,249],"52.38%":[199],"against":[200],"most":[202],"recent":[203],"approach":[204,217,237],"EditAS":[205],".":[206],"Inspired":[207],"by":[208,218,246],"findings,":[210],"construct":[212],"simplistic":[214,236],"retrieval-and-repair-enhanced":[215],"LLM-based":[216,281,299],"transforming":[219],"generation":[222,301],"problem":[223],"into":[224],"program":[226],"repair":[227],"task":[228],"retrieved":[230],"similar":[231],"assertions.":[232],"Surprisingly,":[233],"such":[234],"can":[238],"further":[239,285],"improve":[240],"9.40%":[247],"leading":[250],"new":[252],"records":[253],"datasets.":[256],"provide":[259],"additional":[260],"discussions":[261],"different":[263],"aspects":[264],"(e.g.,":[265,290],"impact":[267],"types":[270],"lengths)":[273],"illustrate":[275],"capacity":[277],"limitations":[279],"approaches.":[282],"Finally,":[283],"pinpoint":[286],"practical":[288,337],"guidelines":[289],"multiple":[294],"candidate":[295],"assertions)":[296],"advanced":[298],"near":[304],"future.":[305],"Overall,":[306],"our":[307],"work":[308],"underscores":[309],"promising":[311],"future":[312],"adopting":[314],"off-the-shelf":[315],"accurate":[319],"meaningful":[321],"reduce":[328],"efforts":[331],"scenarios.":[338]},"counts_by_year":[{"year":2026,"cited_by_count":6},{"year":2025,"cited_by_count":10}],"updated_date":"2026-05-21T06:26:12.895304","created_date":"2025-10-10T00:00:00"}