{"id":"https://openalex.org/W4212769984","doi":"https://doi.org/10.1145/3511430.3511433","title":"Human-based Test Design versus Automated Test Generation: A Literature Review and Meta-Analysis","display_name":"Human-based Test Design versus Automated Test Generation: A Literature Review and Meta-Analysis","publication_year":2022,"publication_date":"2022-02-15","ids":{"openalex":"https://openalex.org/W4212769984","doi":"https://doi.org/10.1145/3511430.3511433"},"language":"en","primary_location":{"id":"doi:10.1145/3511430.3511433","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3511430.3511433","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3511430.3511433","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"15th Innovations in Software Engineering Conference","raw_type":"proceedings-article"},"type":"review","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3511430.3511433","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5016262009","display_name":"Ted Kurmaku","orcid":null},"institutions":[{"id":"https://openalex.org/I82509713","display_name":"M\u00e4lardalen University","ror":"https://ror.org/033vfbz75","country_code":"SE","type":"education","lineage":["https://openalex.org/I82509713"]}],"countries":["SE"],"is_corresponding":true,"raw_author_name":"Ted Kurmaku","raw_affiliation_strings":["Malardalen University, Sweden"],"affiliations":[{"raw_affiliation_string":"Malardalen University, Sweden","institution_ids":["https://openalex.org/I82509713"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5016735339","display_name":"Eduard Paul Enoiu","orcid":"https://orcid.org/0000-0003-2416-4205"},"institutions":[{"id":"https://openalex.org/I82509713","display_name":"M\u00e4lardalen University","ror":"https://ror.org/033vfbz75","country_code":"SE","type":"education","lineage":["https://openalex.org/I82509713"]}],"countries":["SE"],"is_corresponding":false,"raw_author_name":"Eduard Paul Enoiu","raw_affiliation_strings":["Malardalen University, Sweden"],"affiliations":[{"raw_affiliation_string":"Malardalen University, Sweden","institution_ids":["https://openalex.org/I82509713"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5005605535","display_name":"Musa Kumrija","orcid":null},"institutions":[{"id":"https://openalex.org/I82509713","display_name":"M\u00e4lardalen University","ror":"https://ror.org/033vfbz75","country_code":"SE","type":"education","lineage":["https://openalex.org/I82509713"]}],"countries":["SE"],"is_corresponding":false,"raw_author_name":"Musa Kumrija","raw_affiliation_strings":["Malardalen University, Sweden"],"affiliations":[{"raw_affiliation_string":"Malardalen University, Sweden","institution_ids":["https://openalex.org/I82509713"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5016262009"],"corresponding_institution_ids":["https://openalex.org/I82509713"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.01119751,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"11"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12423","display_name":"Software Reliability and Analysis Research","score":0.9983999729156494,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7647525072097778},{"id":"https://openalex.org/keywords/test","display_name":"Test (biology)","score":0.6984941959381104},{"id":"https://openalex.org/keywords/keyword-driven-testing","display_name":"Keyword-driven testing","score":0.6010862588882446},{"id":"https://openalex.org/keywords/test-suite","display_name":"Test suite","score":0.589133620262146},{"id":"https://openalex.org/keywords/test-management-approach","display_name":"Test Management Approach","score":0.5549043416976929},{"id":"https://openalex.org/keywords/reliability-engineering","display_name":"Reliability engineering","score":0.45297685265541077},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.42545393109321594},{"id":"https://openalex.org/keywords/complement","display_name":"Complement (music)","score":0.424201637506485},{"id":"https://openalex.org/keywords/test-case","display_name":"Test case","score":0.4007722735404968},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.37611904740333557},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.34000566601753235},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.19158527255058289},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.17705786228179932},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.10878744721412659}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7647525072097778},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.6984941959381104},{"id":"https://openalex.org/C169168650","wikidata":"https://www.wikidata.org/wiki/Q1675637","display_name":"Keyword-driven testing","level":5,"score":0.6010862588882446},{"id":"https://openalex.org/C151552104","wikidata":"https://www.wikidata.org/wiki/Q7705809","display_name":"Test suite","level":4,"score":0.589133620262146},{"id":"https://openalex.org/C7435765","wikidata":"https://www.wikidata.org/wiki/Q7705776","display_name":"Test Management Approach","level":5,"score":0.5549043416976929},{"id":"https://openalex.org/C200601418","wikidata":"https://www.wikidata.org/wiki/Q2193887","display_name":"Reliability engineering","level":1,"score":0.45297685265541077},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.42545393109321594},{"id":"https://openalex.org/C112313634","wikidata":"https://www.wikidata.org/wiki/Q7886648","display_name":"Complement (music)","level":5,"score":0.424201637506485},{"id":"https://openalex.org/C128942645","wikidata":"https://www.wikidata.org/wiki/Q1568346","display_name":"Test case","level":3,"score":0.4007722735404968},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.37611904740333557},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.34000566601753235},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.19158527255058289},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.17705786228179932},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.10878744721412659},{"id":"https://openalex.org/C152877465","wikidata":"https://www.wikidata.org/wiki/Q208042","display_name":"Regression analysis","level":2,"score":0.0},{"id":"https://openalex.org/C55493867","wikidata":"https://www.wikidata.org/wiki/Q7094","display_name":"Biochemistry","level":1,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C127716648","wikidata":"https://www.wikidata.org/wiki/Q104053","display_name":"Phenotype","level":3,"score":0.0},{"id":"https://openalex.org/C186846655","wikidata":"https://www.wikidata.org/wiki/Q3398377","display_name":"Software construction","level":4,"score":0.0},{"id":"https://openalex.org/C149091818","wikidata":"https://www.wikidata.org/wiki/Q2429814","display_name":"Software system","level":3,"score":0.0},{"id":"https://openalex.org/C104317684","wikidata":"https://www.wikidata.org/wiki/Q7187","display_name":"Gene","level":2,"score":0.0},{"id":"https://openalex.org/C188082640","wikidata":"https://www.wikidata.org/wiki/Q1780899","display_name":"Complementation","level":4,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3511430.3511433","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3511430.3511433","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3511430.3511433","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"15th Innovations in Software Engineering Conference","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3511430.3511433","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3511430.3511433","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3511430.3511433","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"15th Innovations in Software Engineering Conference","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G4726379227","display_name":null,"funder_award_id":"SmartDelta","funder_id":"https://openalex.org/F4320321030","funder_display_name":"VINNOVA"},{"id":"https://openalex.org/G4956428346","display_name":null,"funder_award_id":"Horizon 2020 research and innovatio","funder_id":"https://openalex.org/F4320320300","funder_display_name":"European Commission"},{"id":"https://openalex.org/G5036817778","display_name":null,"funder_award_id":"European Union's Horizon 2020 research and innov","funder_id":"https://openalex.org/F4320320300","funder_display_name":"European Commission"},{"id":"https://openalex.org/G6628542138","display_name":null,"funder_award_id":"957212","funder_id":"https://openalex.org/F4320320300","funder_display_name":"European Commission"},{"id":"https://openalex.org/G8318064016","display_name":null,"funder_award_id":"Horizon","funder_id":"https://openalex.org/F4320320300","funder_display_name":"European Commission"},{"id":"https://openalex.org/G8633428685","display_name":null,"funder_award_id":"European Union's Horizon 2020 research and innovat","funder_id":"https://openalex.org/F4320320300","funder_display_name":"European Commission"},{"id":"https://openalex.org/G8745386224","display_name":null,"funder_award_id":"957212","funder_id":"https://openalex.org/F4320332999","funder_display_name":"Horizon 2020 Framework Programme"}],"funders":[{"id":"https://openalex.org/F4320320300","display_name":"European Commission","ror":"https://ror.org/00k4n6c32"},{"id":"https://openalex.org/F4320321030","display_name":"VINNOVA","ror":"https://ror.org/01kd5m353"},{"id":"https://openalex.org/F4320332999","display_name":"Horizon 2020 Framework Programme","ror":"https://ror.org/00k4n6c32"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4212769984.pdf","grobid_xml":"https://content.openalex.org/works/W4212769984.grobid-xml"},"referenced_works_count":35,"referenced_works":["https://openalex.org/W1607346072","https://openalex.org/W1969156358","https://openalex.org/W1979345446","https://openalex.org/W1980768554","https://openalex.org/W1991901875","https://openalex.org/W1992320755","https://openalex.org/W1994433771","https://openalex.org/W2000383465","https://openalex.org/W2015718800","https://openalex.org/W2027376813","https://openalex.org/W2032947860","https://openalex.org/W2034846556","https://openalex.org/W2038622838","https://openalex.org/W2047670376","https://openalex.org/W2049695835","https://openalex.org/W2055755944","https://openalex.org/W2058554556","https://openalex.org/W2074891852","https://openalex.org/W2089175932","https://openalex.org/W2094738656","https://openalex.org/W2106191864","https://openalex.org/W2113035899","https://openalex.org/W2127424608","https://openalex.org/W2168986002","https://openalex.org/W2267952752","https://openalex.org/W2485305062","https://openalex.org/W2495715420","https://openalex.org/W2496818106","https://openalex.org/W2601283044","https://openalex.org/W2613553457","https://openalex.org/W2882984136","https://openalex.org/W2955667094","https://openalex.org/W2963139460","https://openalex.org/W3007533873","https://openalex.org/W4206221818"],"related_works":["https://openalex.org/W2243231242","https://openalex.org/W2179621094","https://openalex.org/W1978406750","https://openalex.org/W2018145554","https://openalex.org/W1598160211","https://openalex.org/W2497772001","https://openalex.org/W2122987706","https://openalex.org/W2049871268","https://openalex.org/W2204156854","https://openalex.org/W3175215928"],"abstract_inverted_index":{"Automated":[0],"test":[1,8,34,46,55,72,104,108,132,146,198,230,254],"generation":[2,47,73,105,147,231,255],"has":[3,19],"been":[4,20],"proposed":[5],"to":[6,10,26,40,50,93,120,236],"allow":[7],"cases":[9,56],"be":[11,216],"created":[12,160,195],"with":[13,127],"less":[14],"effort.":[15],"While":[16],"much":[17],"progress":[18],"made,":[21],"it":[22],"remains":[23],"a":[24,115,205,219],"challenge":[25],"automatically":[27,128],"generate":[28],"strong":[29],"as":[30,32],"well":[31],"small":[33],"suites":[35],"that":[36,144,172,203,214,250],"are":[37,185,225,256],"also":[38],"relevant":[39],"engineers.":[41],"However,":[42],"how":[43],"these":[44],"automated":[45,71,103,145,197,229,253],"approaches":[48],"compare":[49],"or":[51],"complement":[52],"manually":[53,124,173],"written":[54,125,174],"is":[57],"still":[58],"an":[59],"open":[60],"research":[61],"question.":[62],"In":[63],"the":[64,67,80,89,96,140,156,162,169,237],"light":[65],"of":[66,70,83,99,139,153,158,168,239],"potential":[68],"benefits":[69],"in":[74,151,218],"practice,":[75],"its":[76,87],"long":[77],"history,":[78],"and":[79,106,118,134,161,181,188,222,232,243,252],"apparent":[81],"lack":[82,238],"summative":[84],"evidence":[85],"supporting":[86],"use,":[88],"present":[90],"study":[91],"aims":[92],"systematically":[94],"review":[95,117,142],"current":[97],"body":[98],"peer-reviewed":[100],"publications":[101],"comparing":[102,123,228],"manual":[107,149,233,251],"design":[109],"performed":[110],"by":[111],"humans.":[112],"We":[113],"conducted":[114],"literature":[116,141],"meta-analysis":[119,247],"collect":[121],"data":[122,242],"tests":[126,159,175],"generated":[129],"ones":[130],"regarding":[131],"efficiency":[133],"effectiveness.":[135],"The":[136],"overall":[137],"results":[138,201,224,248],"suggest":[143,202,249],"outperforms":[148],"testing":[150,154,234,260],"terms":[152],"time,":[155],"number":[157],"code":[163],"coverage":[164],"achieved.":[165],"Nevertheless,":[166,245],"most":[167],"studies":[170,207],"report":[171,208],"detect":[176,189],"more":[177,186,190],"faults":[178],"(both":[179],"injected":[180],"naturally":[182],"occurring":[183],"ones),":[184],"readable,":[187],"specific":[191,209],"bugs":[192],"than":[193],"those":[194],"using":[196],"generation.":[199],"Our":[200],"just":[204],"few":[206],"statistics":[210],"(e.g.,":[211],"effect":[212],"sizes)":[213],"can":[215],"used":[217],"proper":[220],"meta-analysis,":[221],"therefore,":[223],"inconclusive":[226],"when":[227],"due":[235],"sufficient":[240],"statistical":[241],"power.":[244],"our":[246],"clearly":[257],"outperforming":[258],"random":[259],"for":[261],"all":[262],"metrics":[263],"considered.":[264]},"counts_by_year":[],"updated_date":"2026-03-11T14:59:36.786465","created_date":"2025-10-10T00:00:00"}