{"id":"https://openalex.org/W4393178049","doi":"https://doi.org/10.1109/access.2024.3391815","title":"Comprehensive Evaluation and Insights Into the Use of Large Language Models in the Automation of Behavior-Driven Development Acceptance Test Formulation","display_name":"Comprehensive Evaluation and Insights Into the Use of Large Language Models in the Automation of Behavior-Driven Development Acceptance Test Formulation","publication_year":2024,"publication_date":"2024-01-01","ids":{"openalex":"https://openalex.org/W4393178049","doi":"https://doi.org/10.1109/access.2024.3391815"},"language":"en","primary_location":{"id":"doi:10.1109/access.2024.3391815","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2024.3391815","pdf_url":"https://ieeexplore.ieee.org/ielx7/6287639/6514899/10506519.pdf","source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},"type":"article","indexed_in":["arxiv","crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://ieeexplore.ieee.org/ielx7/6287639/6514899/10506519.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5072483714","display_name":"Shanthi Karpurapu","orcid":"https://orcid.org/0009-0004-0957-2010"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shanthi Karpurapu","raw_affiliation_strings":["Digital Innovation, SQAC, Ashburn, VA, USA"],"raw_orcid":"https://orcid.org/0009-0004-0957-2010","affiliations":[{"raw_affiliation_string":"Digital Innovation, SQAC, Ashburn, VA, USA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5094246397","display_name":"Sravanthy Myneni","orcid":null},"institutions":[{"id":"https://openalex.org/I180949307","display_name":"Illinois Institute of Technology","ror":"https://ror.org/037t3ry66","country_code":"US","type":"education","lineage":["https://openalex.org/I180949307"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Sravanthy Myneni","raw_affiliation_strings":["Department of Information Technology and Management, Illinois Institute of Technology, Chicago, IL, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Information Technology and Management, Illinois Institute of Technology, Chicago, IL, USA","institution_ids":["https://openalex.org/I180949307"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5094246398","display_name":"Unnati Nettur","orcid":"https://orcid.org/0009-0000-3476-4656"},"institutions":[{"id":"https://openalex.org/I859038795","display_name":"Virginia Tech","ror":"https://ror.org/02smfhw86","country_code":"US","type":"education","lineage":["https://openalex.org/I859038795"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Unnati Nettur","raw_affiliation_strings":["Department of Computer Science, Virginia Tech, Blacksburg, VA, USA"],"raw_orcid":"https://orcid.org/0009-0000-3476-4656","affiliations":[{"raw_affiliation_string":"Department of Computer Science, Virginia Tech, Blacksburg, VA, USA","institution_ids":["https://openalex.org/I859038795"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5094246399","display_name":"Likhit Sagar Gajja","orcid":"https://orcid.org/0009-0001-4037-9436"},"institutions":[{"id":"https://openalex.org/I1323093577","display_name":"BML Munjal University","ror":"https://ror.org/058ay3j75","country_code":"IN","type":"education","lineage":["https://openalex.org/I1323093577"]}],"countries":["IN"],"is_corresponding":false,"raw_author_name":"Likhit Sagar Gajja","raw_affiliation_strings":["Department of Computer Science, BML Munjal University, Gurugram, Haryana, India"],"raw_orcid":"https://orcid.org/0009-0001-4037-9436","affiliations":[{"raw_affiliation_string":"Department of Computer Science, BML Munjal University, Gurugram, Haryana, India","institution_ids":["https://openalex.org/I1323093577"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5080875216","display_name":"Dave Burke","orcid":null},"institutions":[{"id":"https://openalex.org/I2800468613","display_name":"Inova Fairfax Hospital","ror":"https://ror.org/0212h5y77","country_code":"US","type":"healthcare","lineage":["https://openalex.org/I116530498","https://openalex.org/I2800468613"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Dave Burke","raw_affiliation_strings":["Coveros Inc., Fairfax, VA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Coveros Inc., Fairfax, VA, USA","institution_ids":["https://openalex.org/I2800468613"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5094246400","display_name":"Tom Stiehm","orcid":null},"institutions":[{"id":"https://openalex.org/I2800468613","display_name":"Inova Fairfax Hospital","ror":"https://ror.org/0212h5y77","country_code":"US","type":"healthcare","lineage":["https://openalex.org/I116530498","https://openalex.org/I2800468613"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Tom Stiehm","raw_affiliation_strings":["Coveros Inc., Fairfax, VA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Coveros Inc., Fairfax, VA, USA","institution_ids":["https://openalex.org/I2800468613"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5111881173","display_name":"Jeffery Payne","orcid":null},"institutions":[{"id":"https://openalex.org/I2800468613","display_name":"Inova Fairfax Hospital","ror":"https://ror.org/0212h5y77","country_code":"US","type":"healthcare","lineage":["https://openalex.org/I116530498","https://openalex.org/I2800468613"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jeffery Payne","raw_affiliation_strings":["Coveros Inc., Fairfax, VA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Coveros Inc., Fairfax, VA, USA","institution_ids":["https://openalex.org/I2800468613"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":{"value":2075,"currency":"USD","value_usd":2075},"apc_paid":{"value":2075,"currency":"USD","value_usd":2075},"fwci":17.8929,"has_fulltext":true,"cited_by_count":27,"citation_normalized_percentile":{"value":0.99081048,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":100},"biblio":{"volume":"12","issue":null,"first_page":"58715","last_page":"58721"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12423","display_name":"Software Reliability and Analysis Research","score":0.9912999868392944,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10430","display_name":"Software Engineering Techniques and Practices","score":0.9839000105857849,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/agile-software-development","display_name":"Agile software development","score":0.6461685299873352},{"id":"https://openalex.org/keywords/test","display_name":"Test (biology)","score":0.5948900580406189},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5923702716827393},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.5895748734474182},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.4715571403503418},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.36007076501846313},{"id":"https://openalex.org/keywords/process-management","display_name":"Process management","score":0.3434804677963257},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.19145077466964722},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.14180275797843933}],"concepts":[{"id":"https://openalex.org/C14185376","wikidata":"https://www.wikidata.org/wiki/Q30232","display_name":"Agile software development","level":2,"score":0.6461685299873352},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.5948900580406189},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5923702716827393},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.5895748734474182},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.4715571403503418},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.36007076501846313},{"id":"https://openalex.org/C195094911","wikidata":"https://www.wikidata.org/wiki/Q14167904","display_name":"Process management","level":1,"score":0.3434804677963257},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.19145077466964722},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.14180275797843933},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1109/access.2024.3391815","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2024.3391815","pdf_url":"https://ieeexplore.ieee.org/ielx7/6287639/6514899/10506519.pdf","source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},{"id":"pmh:oai:arXiv.org:2403.14965","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2403.14965","pdf_url":"https://arxiv.org/pdf/2403.14965","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:doaj.org/article:df276fddb1184670b412c2c9ba6f72e2","is_oa":true,"landing_page_url":"https://doaj.org/article/df276fddb1184670b412c2c9ba6f72e2","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"IEEE Access, Vol 12, Pp 58715-58721 (2024)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.1109/access.2024.3391815","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2024.3391815","pdf_url":"https://ieeexplore.ieee.org/ielx7/6287639/6514899/10506519.pdf","source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4393178049.pdf","grobid_xml":"https://content.openalex.org/works/W4393178049.grobid-xml"},"referenced_works_count":29,"referenced_works":["https://openalex.org/W1987622710","https://openalex.org/W2060538844","https://openalex.org/W2607213010","https://openalex.org/W2781696492","https://openalex.org/W2793776506","https://openalex.org/W2910453440","https://openalex.org/W3031657470","https://openalex.org/W3080457372","https://openalex.org/W3162543111","https://openalex.org/W3182620217","https://openalex.org/W3185198889","https://openalex.org/W4210447404","https://openalex.org/W4214655501","https://openalex.org/W4319163914","https://openalex.org/W4375840656","https://openalex.org/W4385567149","https://openalex.org/W4385570371","https://openalex.org/W4385627008","https://openalex.org/W4387608638","https://openalex.org/W4387898288","https://openalex.org/W6740580305","https://openalex.org/W6768817161","https://openalex.org/W6772383348","https://openalex.org/W6778883912","https://openalex.org/W6800875267","https://openalex.org/W6810081322","https://openalex.org/W6850625674","https://openalex.org/W6852800892","https://openalex.org/W6854866820"],"related_works":["https://openalex.org/W2656997359","https://openalex.org/W4240110559","https://openalex.org/W2094372386","https://openalex.org/W1973385172","https://openalex.org/W4319430762","https://openalex.org/W2294820933","https://openalex.org/W4281737987","https://openalex.org/W4386449603","https://openalex.org/W3124412501","https://openalex.org/W2048235121"],"abstract_inverted_index":{"Behavior-driven":[0],"development":[1],"(BDD)":[2],"is":[3],"an":[4],"Agile":[5],"testing":[6],"methodology":[7,60],"fostering":[8],"collaboration":[9],"among":[10],"developers,":[11],"QA":[12],"analysts,":[13],"and":[14,42,53,68,77,112,146],"stakeholders.":[15],"In":[16],"this":[17,139],"manuscript,":[18],"we":[19],"propose":[20],"a":[21,58],"novel":[22],"approach":[23,140],"to":[24,33,45,94,132],"enhance":[25],"BDD":[26,81,122,144,154],"practices":[27],"using":[28,158],"large":[29],"language":[30],"models":[31],"(LLMs)":[32],"automate":[34],"acceptance":[35,82,155],"test":[36,156],"generation.":[37],"Our":[38],"study":[39,106,126],"uses":[40],"zero":[41],"few-shot":[43,88],"prompts":[44],"evaluate":[46],"LLMs":[47],"such":[48],"as":[49],"GPT-3.5,":[50],"GPT-4,":[51],"Llama-2-13B,":[52],"PaLM-2.":[54],"The":[55,72,87],"paper":[56],"presents":[57],"detailed":[59],"that":[61,75,128,138],"includes":[62],"the":[63,69,105,133],"dataset,":[64],"prompt":[65,89],"techniques,":[66],"LLMs,":[67,116],"evaluation":[70],"process.":[71],"results":[73],"demonstrate":[74],"GPT-3.5":[76],"GPT-4":[78],"generate":[79],"error-free":[80],"tests":[83],"with":[84],"better":[85],"performance.":[86],"technique":[90],"highlights":[91],"its":[92],"ability":[93],"provide":[95],"higher":[96],"accuracy":[97],"by":[98],"incorporating":[99],"examples":[100],"for":[101,149],"in-context":[102],"learning.":[103],"Furthermore,":[104],"examines":[107],"syntax":[108],"errors,":[109],"validation":[110],"accuracy,":[111],"comparative":[113],"analysis":[114],"of":[115],"revealing":[117],"their":[118],"effectiveness":[119],"in":[120],"enhancing":[121],"practices.":[123],"However,":[124],"our":[125],"acknowledges":[127],"there":[129],"are":[130],"limitations":[131],"proposed":[134],"approach.":[135],"We":[136],"emphasize":[137],"can":[141],"support":[142],"collaborative":[143],"processes":[145],"create":[147],"opportunities":[148],"future":[150],"research":[151],"into":[152],"automated":[153],"generation":[157],"LLMs.":[159]},"counts_by_year":[{"year":2026,"cited_by_count":9},{"year":2025,"cited_by_count":16},{"year":2024,"cited_by_count":2}],"updated_date":"2026-08-26T07:47:46.906454","created_date":"2025-10-10T00:00:00"}