{"id":"https://openalex.org/W3133855622","doi":"https://doi.org/10.22152/programming-journal.org/2021/5/13","title":"Finding Bugs with Specification-Based Testing is Easy!","display_name":"Finding Bugs with Specification-Based Testing is Easy!","publication_year":2021,"publication_date":"2021-02-26","ids":{"openalex":"https://openalex.org/W3133855622","doi":"https://doi.org/10.22152/programming-journal.org/2021/5/13","mag":"3133855622"},"language":"en","primary_location":{"id":"doi:10.22152/programming-journal.org/2021/5/13","is_oa":true,"landing_page_url":"https://doi.org/10.22152/programming-journal.org/2021/5/13","pdf_url":"https://arxiv.org/pdf/2103.00032v1","source":{"id":"https://openalex.org/S4210199781","display_name":"The Art Science and Engineering of Programming","issn_l":"2473-7321","issn":["2473-7321"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"The Art, Science, and Engineering of Programming","raw_type":"journal-article"},"type":"article","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"bronze","oa_url":"https://arxiv.org/pdf/2103.00032v1","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5027757406","display_name":"Janice E. Chin","orcid":null},"institutions":[{"id":"https://openalex.org/I41156924","display_name":"Victoria University of Wellington","ror":"https://ror.org/0040r6f76","country_code":"NZ","type":"education","lineage":["https://openalex.org/I41156924"]}],"countries":["NZ"],"is_corresponding":true,"raw_author_name":"Janice Chin","raw_affiliation_strings":["Victoria University of Wellington, New Zealand"],"affiliations":[{"raw_affiliation_string":"Victoria University of Wellington, New Zealand","institution_ids":["https://openalex.org/I41156924"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5008538442","display_name":"David J. Pearce","orcid":"https://orcid.org/0000-0003-4535-9677"},"institutions":[{"id":"https://openalex.org/I41156924","display_name":"Victoria University of Wellington","ror":"https://ror.org/0040r6f76","country_code":"NZ","type":"education","lineage":["https://openalex.org/I41156924"]}],"countries":["NZ"],"is_corresponding":false,"raw_author_name":"David Pearce","raw_affiliation_strings":["Victoria University of Wellington, New Zealand"],"affiliations":[{"raw_affiliation_string":"Victoria University of Wellington, New Zealand","institution_ids":["https://openalex.org/I41156924"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":["https://openalex.org/A5027757406"],"corresponding_institution_ids":["https://openalex.org/I41156924"],"apc_list":null,"apc_paid":null,"fwci":0.6321,"has_fulltext":true,"cited_by_count":3,"citation_normalized_percentile":{"value":0.68554944,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":91,"max":96},"biblio":{"volume":"5","issue":"3","first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9984999895095825,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12423","display_name":"Software Reliability and Analysis Research","score":0.9980000257492065,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8554186224937439},{"id":"https://openalex.org/keywords/haskell","display_name":"Haskell","score":0.6500831246376038},{"id":"https://openalex.org/keywords/random-testing","display_name":"Random testing","score":0.6222752928733826},{"id":"https://openalex.org/keywords/fuzz-testing","display_name":"Fuzz testing","score":0.6133358478546143},{"id":"https://openalex.org/keywords/suite","display_name":"Suite","score":0.5945236682891846},{"id":"https://openalex.org/keywords/test-suite","display_name":"Test suite","score":0.5916059613227844},{"id":"https://openalex.org/keywords/usability","display_name":"Usability","score":0.5496795773506165},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.5188387632369995},{"id":"https://openalex.org/keywords/non-regression-testing","display_name":"Non-regression testing","score":0.49747541546821594},{"id":"https://openalex.org/keywords/keyword-driven-testing","display_name":"Keyword-driven testing","score":0.487784206867218},{"id":"https://openalex.org/keywords/test-strategy","display_name":"Test strategy","score":0.4804927110671997},{"id":"https://openalex.org/keywords/counterexample","display_name":"Counterexample","score":0.46364179253578186},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.46226561069488525},{"id":"https://openalex.org/keywords/software-performance-testing","display_name":"Software performance testing","score":0.4454192817211151},{"id":"https://openalex.org/keywords/integration-testing","display_name":"Integration testing","score":0.4117025136947632},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.41112473607063293},{"id":"https://openalex.org/keywords/test-case","display_name":"Test case","score":0.3864072561264038},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.2564748525619507},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.20833179354667664},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.1377480924129486},{"id":"https://openalex.org/keywords/software-development","display_name":"Software development","score":0.13133928179740906},{"id":"https://openalex.org/keywords/software-construction","display_name":"Software construction","score":0.10043919086456299}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8554186224937439},{"id":"https://openalex.org/C2780624054","wikidata":"https://www.wikidata.org/wiki/Q34010","display_name":"Haskell","level":3,"score":0.6500831246376038},{"id":"https://openalex.org/C106159264","wikidata":"https://www.wikidata.org/wiki/Q17146789","display_name":"Random testing","level":4,"score":0.6222752928733826},{"id":"https://openalex.org/C111065885","wikidata":"https://www.wikidata.org/wiki/Q1189053","display_name":"Fuzz testing","level":3,"score":0.6133358478546143},{"id":"https://openalex.org/C79581498","wikidata":"https://www.wikidata.org/wiki/Q1367530","display_name":"Suite","level":2,"score":0.5945236682891846},{"id":"https://openalex.org/C151552104","wikidata":"https://www.wikidata.org/wiki/Q7705809","display_name":"Test suite","level":4,"score":0.5916059613227844},{"id":"https://openalex.org/C170130773","wikidata":"https://www.wikidata.org/wiki/Q216378","display_name":"Usability","level":2,"score":0.5496795773506165},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.5188387632369995},{"id":"https://openalex.org/C86469151","wikidata":"https://www.wikidata.org/wiki/Q917415","display_name":"Non-regression testing","level":5,"score":0.49747541546821594},{"id":"https://openalex.org/C169168650","wikidata":"https://www.wikidata.org/wiki/Q1675637","display_name":"Keyword-driven testing","level":5,"score":0.487784206867218},{"id":"https://openalex.org/C188598960","wikidata":"https://www.wikidata.org/wiki/Q7705805","display_name":"Test strategy","level":3,"score":0.4804927110671997},{"id":"https://openalex.org/C162838799","wikidata":"https://www.wikidata.org/wiki/Q596077","display_name":"Counterexample","level":2,"score":0.46364179253578186},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.46226561069488525},{"id":"https://openalex.org/C178059732","wikidata":"https://www.wikidata.org/wiki/Q1982529","display_name":"Software performance testing","level":5,"score":0.4454192817211151},{"id":"https://openalex.org/C107683887","wikidata":"https://www.wikidata.org/wiki/Q782466","display_name":"Integration testing","level":3,"score":0.4117025136947632},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.41112473607063293},{"id":"https://openalex.org/C128942645","wikidata":"https://www.wikidata.org/wiki/Q1568346","display_name":"Test case","level":3,"score":0.3864072561264038},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.2564748525619507},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.20833179354667664},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.1377480924129486},{"id":"https://openalex.org/C529173508","wikidata":"https://www.wikidata.org/wiki/Q638608","display_name":"Software development","level":3,"score":0.13133928179740906},{"id":"https://openalex.org/C186846655","wikidata":"https://www.wikidata.org/wiki/Q3398377","display_name":"Software construction","level":4,"score":0.10043919086456299},{"id":"https://openalex.org/C95457728","wikidata":"https://www.wikidata.org/wiki/Q309","display_name":"History","level":0,"score":0.0},{"id":"https://openalex.org/C118615104","wikidata":"https://www.wikidata.org/wiki/Q121416","display_name":"Discrete mathematics","level":1,"score":0.0},{"id":"https://openalex.org/C152877465","wikidata":"https://www.wikidata.org/wiki/Q208042","display_name":"Regression analysis","level":2,"score":0.0},{"id":"https://openalex.org/C166957645","wikidata":"https://www.wikidata.org/wiki/Q23498","display_name":"Archaeology","level":1,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0},{"id":"https://openalex.org/C42383842","wikidata":"https://www.wikidata.org/wiki/Q193076","display_name":"Functional programming","level":2,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.22152/programming-journal.org/2021/5/13","is_oa":true,"landing_page_url":"https://doi.org/10.22152/programming-journal.org/2021/5/13","pdf_url":"https://arxiv.org/pdf/2103.00032v1","source":{"id":"https://openalex.org/S4210199781","display_name":"The Art Science and Engineering of Programming","issn_l":"2473-7321","issn":["2473-7321"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"The Art, Science, and Engineering of Programming","raw_type":"journal-article"},{"id":"pmh:oai:arXiv.org:2103.00032","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2103.00032","pdf_url":"https://arxiv.org/pdf/2103.00032","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"}],"best_oa_location":{"id":"doi:10.22152/programming-journal.org/2021/5/13","is_oa":true,"landing_page_url":"https://doi.org/10.22152/programming-journal.org/2021/5/13","pdf_url":"https://arxiv.org/pdf/2103.00032v1","source":{"id":"https://openalex.org/S4210199781","display_name":"The Art Science and Engineering of Programming","issn_l":"2473-7321","issn":["2473-7321"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"The Art, Science, and Engineering of Programming","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3133855622.pdf","grobid_xml":"https://content.openalex.org/works/W3133855622.grobid-xml"},"referenced_works_count":119,"referenced_works":["https://openalex.org/W16331198","https://openalex.org/W39595256","https://openalex.org/W59502829","https://openalex.org/W73173988","https://openalex.org/W104102279","https://openalex.org/W127773936","https://openalex.org/W179270291","https://openalex.org/W1495368328","https://openalex.org/W1504917336","https://openalex.org/W1507029499","https://openalex.org/W1509650477","https://openalex.org/W1511434008","https://openalex.org/W1522925810","https://openalex.org/W1523445204","https://openalex.org/W1537084112","https://openalex.org/W1542690313","https://openalex.org/W1545681762","https://openalex.org/W1548974835","https://openalex.org/W1549159967","https://openalex.org/W1553352239","https://openalex.org/W1553421328","https://openalex.org/W1555693494","https://openalex.org/W1565541828","https://openalex.org/W1580328473","https://openalex.org/W1606142489","https://openalex.org/W1611541577","https://openalex.org/W1616208858","https://openalex.org/W1820726602","https://openalex.org/W1959256509","https://openalex.org/W1965194038","https://openalex.org/W1971650562","https://openalex.org/W1972085995","https://openalex.org/W1979116650","https://openalex.org/W1979619055","https://openalex.org/W1981663184","https://openalex.org/W1993255342","https://openalex.org/W1993836075","https://openalex.org/W1998838326","https://openalex.org/W2000691319","https://openalex.org/W2009489720","https://openalex.org/W2024219209","https://openalex.org/W2024442685","https://openalex.org/W2030225697","https://openalex.org/W2030358379","https://openalex.org/W2031872041","https://openalex.org/W2035529004","https://openalex.org/W2045060485","https://openalex.org/W2051181744","https://openalex.org/W2060795628","https://openalex.org/W2067636118","https://openalex.org/W2069005612","https://openalex.org/W2074285438","https://openalex.org/W2095473904","https://openalex.org/W2096425305","https://openalex.org/W2103999647","https://openalex.org/W2107709519","https://openalex.org/W2111654822","https://openalex.org/W2113175210","https://openalex.org/W2113944334","https://openalex.org/W2114337691","https://openalex.org/W2119696713","https://openalex.org/W2119717320","https://openalex.org/W2119885577","https://openalex.org/W2125940008","https://openalex.org/W2127053650","https://openalex.org/W2130427425","https://openalex.org/W2132296233","https://openalex.org/W2132980721","https://openalex.org/W2137086189","https://openalex.org/W2140927311","https://openalex.org/W2143889043","https://openalex.org/W2146972408","https://openalex.org/W2149207009","https://openalex.org/W2150684130","https://openalex.org/W2152949134","https://openalex.org/W2162120832","https://openalex.org/W2162801047","https://openalex.org/W2170898576","https://openalex.org/W2171269531","https://openalex.org/W2172260321","https://openalex.org/W2176407799","https://openalex.org/W2202481953","https://openalex.org/W2278624804","https://openalex.org/W2288908380","https://openalex.org/W2289498755","https://openalex.org/W2291881641","https://openalex.org/W2294722445","https://openalex.org/W2464455595","https://openalex.org/W2476059279","https://openalex.org/W2476404036","https://openalex.org/W2504995592","https://openalex.org/W2522590477","https://openalex.org/W2565255124","https://openalex.org/W2624454096","https://openalex.org/W2746674887","https://openalex.org/W2752061190","https://openalex.org/W2805419234","https://openalex.org/W2810429158","https://openalex.org/W2891612091","https://openalex.org/W2893185619","https://openalex.org/W2936891931","https://openalex.org/W2953254264","https://openalex.org/W2962694250","https://openalex.org/W3006045359","https://openalex.org/W3083214190","https://openalex.org/W3104664063","https://openalex.org/W3146035799","https://openalex.org/W4205962651","https://openalex.org/W4234542549","https://openalex.org/W4235130846","https://openalex.org/W4239051178","https://openalex.org/W4243922950","https://openalex.org/W4244488020","https://openalex.org/W4250336630","https://openalex.org/W4253302042","https://openalex.org/W4287901611","https://openalex.org/W4301954000","https://openalex.org/W4381304672","https://openalex.org/W4391285994"],"related_works":["https://openalex.org/W2560445721","https://openalex.org/W3214776400","https://openalex.org/W4389884952","https://openalex.org/W3197709817","https://openalex.org/W2362944210","https://openalex.org/W2915670848","https://openalex.org/W2074050424","https://openalex.org/W2955592476","https://openalex.org/W2023956971","https://openalex.org/W2041379335"],"abstract_inverted_index":{"Automated":[0],"specification-based":[1],"testing":[2,20,61,85,111,159,215],"has":[3,134,247],"a":[4,128,150,164,200,205],"long":[5],"history":[6],"with":[7,93,101,125,228],"several":[8,189],"notable":[9],"tools":[10],"having":[11],"emerged.":[12],"For":[13,118],"example,":[14,119],"QuickCheck":[15],"for":[16,96,112,161,181],"Haskell":[17],"focuses":[18],"on":[19,79,188,238],"against":[21],"user-provided":[22],"properties.":[23],"Others,":[24],"such":[25],"as":[26,254],"JMLUnit,":[27],"use":[28],"specifications":[29],"in":[30,53,89,104,142,218,232,255],"the":[31,136,154,169,173,193,213,243,256],"form":[32],"of":[33,86,153,202,207,242],"pre-and":[34],"post-conditions":[35],"to":[36,171],"drive":[37],"testing.":[38],"An":[39],"interesting":[40],"(and":[41],"under-explored)":[42],"question":[43],"is":[44,49,62,123,130,168,216],"how":[45,72],"effective":[46,64,217,249],"this":[47,80,143],"approach":[48],"at":[50,65,196,250],"finding":[51,67,198,251],"bugs":[52,252],"practice.":[54],"In":[55],"general,":[56],"one":[57],"would":[58],"assume":[59],"automated":[60,84,110,126,158,214],"less":[63,74],"bug":[66,197],"than":[68],"static":[69,102],"verification.":[70],"But,":[71],"much":[73],"effective?":[75],"To":[76,145],"shed":[77],"light":[78],"question,":[81],"we":[82,106,148,177,236],"consider":[83],"programs":[87],"written":[88],"Whiley":[90,113],"-a":[91],"language":[92],"first-class":[94],"support":[95],"specifications.":[97],"Whilst":[98],"originally":[99],"designed":[100],"verification":[103],"mind,":[105],"have":[107],"anecdotally":[108],"found":[109],"surprisingly":[114],"useful":[115,225],"and":[116,176,183,221],"cost-effective.":[117],"when":[120],"an":[121,157],"error":[122],"detected":[124],"testing,":[127],"counterexample":[129],"always":[131],"provided.":[132],"This":[133],"motivated":[135],"more":[137],"rigorous":[138],"empirical":[139],"examination":[140],"presented":[141],"paper.":[144],"that":[146,222],"end,":[147],"provide":[149],"technical":[151],"discussion":[152],"implementation":[155],"behind":[156],"tool":[160,244],"Whiley.":[162],"Here,":[163],"key":[165],"usability":[166],"concern":[167],"ability":[170],"parameterise":[172],"input":[174],"space,":[175],"present":[178],"novel":[179],"approaches":[180],"references":[182],"lambdas.":[184],"We":[185],"then":[186],"report":[187,237],"large":[190],"experiments":[191],"investigating":[192],"tool's":[194],"effectiveness":[195],"using":[199],"range":[201],"benchmarks,":[203],"including":[204],"suite":[206],"1800+":[208],"mutants.":[209],"The":[210],"results":[211],"indicate":[212],"many":[219],"cases,":[220],"sampling":[223],"offers":[224],"performance":[226],"benefits":[227],"only":[229],"modest":[230],"reductions":[231],"bug-finding":[233],"capability.":[234],"Finally,":[235],"some":[239],"real-world":[240],"uses":[241],"where":[245],"it":[246],"proved":[248],"(such":[253],"standard":[257],"library).":[258]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2022,"cited_by_count":2}],"updated_date":"2026-03-20T23:20:44.827607","created_date":"2025-10-10T00:00:00"}