{"id":"https://openalex.org/W2022873981","doi":"https://doi.org/10.1145/2635868.2635923","title":"On the efficiency of automated testing","display_name":"On the efficiency of automated testing","publication_year":2014,"publication_date":"2014-11-04","ids":{"openalex":"https://openalex.org/W2022873981","doi":"https://doi.org/10.1145/2635868.2635923","mag":"2022873981"},"language":"en","primary_location":{"id":"doi:10.1145/2635868.2635923","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2635868.2635923","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 22nd ACM SIGSOFT International Symposium on Foundations of Software Engineering","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5005260100","display_name":"Marcel B\u00f6hme","orcid":"https://orcid.org/0000-0002-4470-1824"},"institutions":[{"id":"https://openalex.org/I91712215","display_name":"Saarland University","ror":"https://ror.org/01jdpyv68","country_code":"DE","type":"education","lineage":["https://openalex.org/I91712215"]}],"countries":["DE"],"is_corresponding":true,"raw_author_name":"Marcel B\u00f6hme","raw_affiliation_strings":["Saarland University, Germany"],"affiliations":[{"raw_affiliation_string":"Saarland University, Germany","institution_ids":["https://openalex.org/I91712215"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100683996","display_name":"Soumya Paul","orcid":"https://orcid.org/0000-0002-0115-4189"},"institutions":[{"id":"https://openalex.org/I165932596","display_name":"National University of Singapore","ror":"https://ror.org/01tgyzw49","country_code":"SG","type":"education","lineage":["https://openalex.org/I165932596"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Soumya Paul","raw_affiliation_strings":["National University of Singapore, Singapore"],"affiliations":[{"raw_affiliation_string":"National University of Singapore, Singapore","institution_ids":["https://openalex.org/I165932596"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":["https://openalex.org/A5005260100"],"corresponding_institution_ids":["https://openalex.org/I91712215"],"apc_list":null,"apc_paid":null,"fwci":4.3676,"has_fulltext":false,"cited_by_count":23,"citation_normalized_percentile":{"value":0.93991641,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"632","last_page":"642"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12423","display_name":"Software Reliability and Analysis Research","score":0.9988999962806702,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9966999888420105,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/correctness","display_name":"Correctness","score":0.8473823070526123},{"id":"https://openalex.org/keywords/random-testing","display_name":"Random testing","score":0.7636934518814087},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6325060725212097},{"id":"https://openalex.org/keywords/sampling","display_name":"Sampling (signal processing)","score":0.49591073393821716},{"id":"https://openalex.org/keywords/upper-and-lower-bounds","display_name":"Upper and lower bounds","score":0.48982998728752136},{"id":"https://openalex.org/keywords/sample","display_name":"Sample (material)","score":0.44253650307655334},{"id":"https://openalex.org/keywords/confidence-interval","display_name":"Confidence interval","score":0.4172241687774658},{"id":"https://openalex.org/keywords/sample-size-determination","display_name":"Sample size determination","score":0.4130100607872009},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.4011302888393402},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.3349039554595947},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.33237284421920776},{"id":"https://openalex.org/keywords/test-case","display_name":"Test case","score":0.24768200516700745},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.10377031564712524}],"concepts":[{"id":"https://openalex.org/C55439883","wikidata":"https://www.wikidata.org/wiki/Q360812","display_name":"Correctness","level":2,"score":0.8473823070526123},{"id":"https://openalex.org/C106159264","wikidata":"https://www.wikidata.org/wiki/Q17146789","display_name":"Random testing","level":4,"score":0.7636934518814087},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6325060725212097},{"id":"https://openalex.org/C140779682","wikidata":"https://www.wikidata.org/wiki/Q210868","display_name":"Sampling (signal processing)","level":3,"score":0.49591073393821716},{"id":"https://openalex.org/C77553402","wikidata":"https://www.wikidata.org/wiki/Q13222579","display_name":"Upper and lower bounds","level":2,"score":0.48982998728752136},{"id":"https://openalex.org/C198531522","wikidata":"https://www.wikidata.org/wiki/Q485146","display_name":"Sample (material)","level":2,"score":0.44253650307655334},{"id":"https://openalex.org/C44249647","wikidata":"https://www.wikidata.org/wiki/Q208498","display_name":"Confidence interval","level":2,"score":0.4172241687774658},{"id":"https://openalex.org/C129848803","wikidata":"https://www.wikidata.org/wiki/Q2564360","display_name":"Sample size determination","level":2,"score":0.4130100607872009},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.4011302888393402},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.3349039554595947},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.33237284421920776},{"id":"https://openalex.org/C128942645","wikidata":"https://www.wikidata.org/wiki/Q1568346","display_name":"Test case","level":3,"score":0.24768200516700745},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.10377031564712524},{"id":"https://openalex.org/C43617362","wikidata":"https://www.wikidata.org/wiki/Q170050","display_name":"Chromatography","level":1,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0},{"id":"https://openalex.org/C152877465","wikidata":"https://www.wikidata.org/wiki/Q208042","display_name":"Regression analysis","level":2,"score":0.0},{"id":"https://openalex.org/C106131492","wikidata":"https://www.wikidata.org/wiki/Q3072260","display_name":"Filter (signal processing)","level":2,"score":0.0},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/2635868.2635923","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2635868.2635923","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 22nd ACM SIGSOFT International Symposium on Foundations of Software Engineering","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":34,"referenced_works":["https://openalex.org/W1600639883","https://openalex.org/W1710734607","https://openalex.org/W1857528250","https://openalex.org/W1880675778","https://openalex.org/W1963523113","https://openalex.org/W1971650562","https://openalex.org/W1974394819","https://openalex.org/W1980818933","https://openalex.org/W2011367000","https://openalex.org/W2012210872","https://openalex.org/W2056309769","https://openalex.org/W2057891174","https://openalex.org/W2060573639","https://openalex.org/W2071112258","https://openalex.org/W2091299652","https://openalex.org/W2096449544","https://openalex.org/W2101629181","https://openalex.org/W2103201337","https://openalex.org/W2110441383","https://openalex.org/W2114772245","https://openalex.org/W2115484804","https://openalex.org/W2117963349","https://openalex.org/W2119249385","https://openalex.org/W2119805979","https://openalex.org/W2125152082","https://openalex.org/W2125539770","https://openalex.org/W2126731841","https://openalex.org/W2134875273","https://openalex.org/W2135841285","https://openalex.org/W2160486231","https://openalex.org/W2162120832","https://openalex.org/W2165723276","https://openalex.org/W4237492309","https://openalex.org/W4253606798"],"related_works":["https://openalex.org/W1667647204","https://openalex.org/W2404647514","https://openalex.org/W4247536566","https://openalex.org/W4241418540","https://openalex.org/W2018477250","https://openalex.org/W3119814709","https://openalex.org/W1508895727","https://openalex.org/W2725786787","https://openalex.org/W1590965489","https://openalex.org/W1578161921"],"abstract_inverted_index":{"The":[0,19],"aim":[1],"of":[2,40,58,90,116,130,153,184,194,237,259],"automated":[3],"program":[4,42,60,179,232],"testing":[5,31,72,103],"is":[6,35,73,119,165,199,253,273],"to":[7,51,121,148,223,228,246,255],"gain":[8],"confidence":[9,176,260],"about":[10],"a":[11,158,283],"program's":[12],"correctness":[13],"by":[14,37],"sampling":[15,20,34],"its":[16,185,238],"input":[17,111],"space.":[18],"process":[21],"can":[22],"be":[23],"either":[24],"systematic":[25,30,91,102],"or":[26,54],"random.":[27],"For":[28,216],"every":[29,108],"technique":[32,104],"the":[33,38,45,47,59,87,99,114,122,128,143,150,175,178,203,231,257,267,270],"informed":[36],"analysis":[39,65,80,123,160],"some":[41,67],"artefacts,":[43],"like":[44],"specification,":[46],"source":[48],"code":[49],"(e.g.,":[50,61],"achieve":[52],"coverage),":[53],"even":[55],"faulty":[56],"versions":[57],"mutation":[62],"testing).":[63],"This":[64],"incurs":[66],"cost.":[68,81],"In":[69,82],"contrast,":[70],"random":[71,93],"unsystematic":[74],"and":[75,118],"does":[76],"not":[77],"sustain":[78],"any":[79],"this":[83,209],"paper,":[84],"we":[85,96,137,140,173,187],"investigate":[86],"theoretical":[88],"efficiency":[89,129],"versus":[92],"testing.":[94],"First,":[95],"mathematically":[97],"model":[98],"most":[100],"effective":[101],"S_0":[105,131,147,240],"in":[106,275,282],"which":[107,163,197],"sampled":[109],"test":[110,226,249],"strictly":[112],"increases":[113],"\"degree":[115],"confidence\"":[117],"subject":[120],"cost":[124,161,268],"c.":[125,134],"Note":[126],"that":[127,172,177,208,230],"depends":[132,211],"on":[133,192,202,214,266],"Specifically,":[135],"if":[136,269],"increase":[138,142],"c,":[139],"also":[141,206],"time":[144,222,285],"it":[145],"takes":[146],"establish":[149,229,256],"same":[151],"degree":[152],"confidence.":[154],"So,":[155],"there":[156],"exists":[157],"maximum":[159],"beyond":[162,196],"R":[164,198,219,252],"generally":[166],"more":[167,200],"efficient":[168,201],"than":[169,244],"S_0.":[170],"Given":[171],"require":[174],"works":[180,233],"correctly":[181,234],"for":[182,235],"x%":[183],"input,":[186,239],"prove":[188,263],"an":[189],"upper":[190],"bound":[191,210],"c":[193],"S_0,":[195],"average.":[204],"We":[205,262],"show":[207],"asymptotically":[212],"only":[213],"x.":[215],"instance,":[217],"let":[218],"take":[220,242],"10ms":[221],"sample":[224,247],"one":[225,248],"input;":[227],"90%":[236],"must":[241],"less":[243],"41ms":[245],"input.":[250],"Otherwise,":[251],"expected":[254],"90%-degree":[258],"earlier.":[261],"similar":[264],"bounds":[265],"software":[271],"tester":[272],"interested":[274],"revealing":[276],"as":[277,280],"many":[278],"errors":[279],"possible":[281],"given":[284],"span.":[286]},"counts_by_year":[{"year":2024,"cited_by_count":1},{"year":2022,"cited_by_count":5},{"year":2021,"cited_by_count":2},{"year":2020,"cited_by_count":3},{"year":2019,"cited_by_count":1},{"year":2018,"cited_by_count":1},{"year":2017,"cited_by_count":4},{"year":2016,"cited_by_count":2},{"year":2015,"cited_by_count":4}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}