{"id":"https://openalex.org/W2087031002","doi":"https://doi.org/10.1142/s0218213000000112","title":"ADEQUACY OF LIMITED TESTING FOR KNOWLEDGE BASED SYSTEMS","display_name":"ADEQUACY OF LIMITED TESTING FOR KNOWLEDGE BASED SYSTEMS","publication_year":2000,"publication_date":"2000-03-01","ids":{"openalex":"https://openalex.org/W2087031002","doi":"https://doi.org/10.1142/s0218213000000112","mag":"2087031002"},"language":"en","primary_location":{"id":"doi:10.1142/s0218213000000112","is_oa":false,"landing_page_url":"https://doi.org/10.1142/s0218213000000112","pdf_url":null,"source":{"id":"https://openalex.org/S178780388","display_name":"International Journal of Artificial Intelligence Tools","issn_l":"0218-2130","issn":["0218-2130","1793-6349"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319815","host_organization_name":"World Scientific","host_organization_lineage":["https://openalex.org/P4310319815"],"host_organization_lineage_names":["World Scientific"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"International Journal on Artificial Intelligence Tools","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5077008083","display_name":"Tim Menzies","orcid":"https://orcid.org/0000-0002-5040-3196"},"institutions":[{"id":"https://openalex.org/I12097938","display_name":"West Virginia University","ror":"https://ror.org/011vxgd24","country_code":"US","type":"education","lineage":["https://openalex.org/I12097938"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"TIM MENZIES","raw_affiliation_strings":["NASA/WVU Software Research Laboratory, Department of Computer Science and Electrical Engineering, West Virginia University, Morgantown, WV 26505, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"NASA/WVU Software Research Laboratory, Department of Computer Science and Electrical Engineering, West Virginia University, Morgantown, WV 26505, USA","institution_ids":["https://openalex.org/I12097938"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5015369081","display_name":"Bojan \u010cuki\u0107","orcid":"https://orcid.org/0000-0001-7130-9054"},"institutions":[{"id":"https://openalex.org/I12097938","display_name":"West Virginia University","ror":"https://ror.org/011vxgd24","country_code":"US","type":"education","lineage":["https://openalex.org/I12097938"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"BOJAN CUKIC","raw_affiliation_strings":["NASA/WVU Software Research Laboratory, Department of Computer Science and Electrical Engineering, West Virginia University, Morgantown, WV 26505, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"NASA/WVU Software Research Laboratory, Department of Computer Science and Electrical Engineering, West Virginia University, Morgantown, WV 26505, USA","institution_ids":["https://openalex.org/I12097938"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I12097938"],"apc_list":null,"apc_paid":null,"fwci":5.7844,"has_fulltext":false,"cited_by_count":26,"citation_normalized_percentile":{"value":0.96007722,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":94},"biblio":{"volume":"09","issue":"01","first_page":"153","last_page":"172"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10906","display_name":"AI-based Problem Solving and Planning","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10906","display_name":"AI-based Problem Solving and Planning","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9904000163078308,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.9887999892234802,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8510651588439941},{"id":"https://openalex.org/keywords/random-testing","display_name":"Random testing","score":0.7084478139877319},{"id":"https://openalex.org/keywords/test-suite","display_name":"Test suite","score":0.6999945640563965},{"id":"https://openalex.org/keywords/completeness","display_name":"Completeness (order theory)","score":0.6957564353942871},{"id":"https://openalex.org/keywords/white-box-testing","display_name":"White-box testing","score":0.532908022403717},{"id":"https://openalex.org/keywords/black-box","display_name":"Black box","score":0.5036136507987976},{"id":"https://openalex.org/keywords/manual-testing","display_name":"Manual testing","score":0.4999558925628662},{"id":"https://openalex.org/keywords/suite","display_name":"Suite","score":0.4819895625114441},{"id":"https://openalex.org/keywords/test-strategy","display_name":"Test strategy","score":0.46412184834480286},{"id":"https://openalex.org/keywords/system-testing","display_name":"System testing","score":0.4602926969528198},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.45985040068626404},{"id":"https://openalex.org/keywords/keyword-driven-testing","display_name":"Keyword-driven testing","score":0.45172446966171265},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.4410465955734253},{"id":"https://openalex.org/keywords/test-case","display_name":"Test case","score":0.42212748527526855},{"id":"https://openalex.org/keywords/system-under-test","display_name":"System under test","score":0.41204833984375},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.3916356563568115},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.32868072390556335},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.27049413323402405},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.1406365931034088},{"id":"https://openalex.org/keywords/software-system","display_name":"Software system","score":0.09553191065788269}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8510651588439941},{"id":"https://openalex.org/C106159264","wikidata":"https://www.wikidata.org/wiki/Q17146789","display_name":"Random testing","level":4,"score":0.7084478139877319},{"id":"https://openalex.org/C151552104","wikidata":"https://www.wikidata.org/wiki/Q7705809","display_name":"Test suite","level":4,"score":0.6999945640563965},{"id":"https://openalex.org/C17231256","wikidata":"https://www.wikidata.org/wiki/Q5156540","display_name":"Completeness (order theory)","level":2,"score":0.6957564353942871},{"id":"https://openalex.org/C162443782","wikidata":"https://www.wikidata.org/wiki/Q1066228","display_name":"White-box testing","level":5,"score":0.532908022403717},{"id":"https://openalex.org/C94966114","wikidata":"https://www.wikidata.org/wiki/Q29256","display_name":"Black box","level":2,"score":0.5036136507987976},{"id":"https://openalex.org/C182122060","wikidata":"https://www.wikidata.org/wiki/Q6752328","display_name":"Manual testing","level":5,"score":0.4999558925628662},{"id":"https://openalex.org/C79581498","wikidata":"https://www.wikidata.org/wiki/Q1367530","display_name":"Suite","level":2,"score":0.4819895625114441},{"id":"https://openalex.org/C188598960","wikidata":"https://www.wikidata.org/wiki/Q7705805","display_name":"Test strategy","level":3,"score":0.46412184834480286},{"id":"https://openalex.org/C7166840","wikidata":"https://www.wikidata.org/wiki/Q1199682","display_name":"System testing","level":2,"score":0.4602926969528198},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.45985040068626404},{"id":"https://openalex.org/C169168650","wikidata":"https://www.wikidata.org/wiki/Q1675637","display_name":"Keyword-driven testing","level":5,"score":0.45172446966171265},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4410465955734253},{"id":"https://openalex.org/C128942645","wikidata":"https://www.wikidata.org/wiki/Q1568346","display_name":"Test case","level":3,"score":0.42212748527526855},{"id":"https://openalex.org/C108913964","wikidata":"https://www.wikidata.org/wiki/Q2376856","display_name":"System under test","level":4,"score":0.41204833984375},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3916356563568115},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.32868072390556335},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.27049413323402405},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.1406365931034088},{"id":"https://openalex.org/C149091818","wikidata":"https://www.wikidata.org/wiki/Q2429814","display_name":"Software system","level":3,"score":0.09553191065788269},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C95457728","wikidata":"https://www.wikidata.org/wiki/Q309","display_name":"History","level":0,"score":0.0},{"id":"https://openalex.org/C186846655","wikidata":"https://www.wikidata.org/wiki/Q3398377","display_name":"Software construction","level":4,"score":0.0},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0},{"id":"https://openalex.org/C152877465","wikidata":"https://www.wikidata.org/wiki/Q208042","display_name":"Regression analysis","level":2,"score":0.0},{"id":"https://openalex.org/C166957645","wikidata":"https://www.wikidata.org/wiki/Q23498","display_name":"Archaeology","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1142/s0218213000000112","is_oa":false,"landing_page_url":"https://doi.org/10.1142/s0218213000000112","pdf_url":null,"source":{"id":"https://openalex.org/S178780388","display_name":"International Journal of Artificial Intelligence Tools","issn_l":"0218-2130","issn":["0218-2130","1793-6349"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319815","host_organization_name":"World Scientific","host_organization_lineage":["https://openalex.org/P4310319815"],"host_organization_lineage_names":["World Scientific"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"International Journal on Artificial Intelligence Tools","raw_type":"journal-article"},{"id":"pmh:oai:CiteSeerX.psu:10.1.1.106.3268","is_oa":false,"landing_page_url":"http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.106.3268","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://menzies.us/pdf/00ijait.pdf","raw_type":"text"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.4099999964237213,"id":"https://metadata.un.org/sdg/17","display_name":"Partnerships for the goals"}],"awards":[],"funders":[{"id":"https://openalex.org/F4320306101","display_name":"National Aeronautics and Space Administration","ror":"https://ror.org/027ka1x80"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":44,"referenced_works":["https://openalex.org/W145815032","https://openalex.org/W180992408","https://openalex.org/W1481278943","https://openalex.org/W1487895014","https://openalex.org/W1512624111","https://openalex.org/W1519683110","https://openalex.org/W1525796185","https://openalex.org/W1536406628","https://openalex.org/W1538017975","https://openalex.org/W1544444365","https://openalex.org/W1564733196","https://openalex.org/W1620410031","https://openalex.org/W1667614912","https://openalex.org/W1696263757","https://openalex.org/W1820930518","https://openalex.org/W1980818933","https://openalex.org/W1984431881","https://openalex.org/W1986766075","https://openalex.org/W1988076559","https://openalex.org/W2008384517","https://openalex.org/W2013336535","https://openalex.org/W2037481186","https://openalex.org/W2037783738","https://openalex.org/W2038842251","https://openalex.org/W2054699624","https://openalex.org/W2060299693","https://openalex.org/W2067990963","https://openalex.org/W2071639741","https://openalex.org/W2072728459","https://openalex.org/W2109676845","https://openalex.org/W2110679033","https://openalex.org/W2113936771","https://openalex.org/W2117189826","https://openalex.org/W2124300773","https://openalex.org/W2136069883","https://openalex.org/W2145287593","https://openalex.org/W2147096558","https://openalex.org/W2149275280","https://openalex.org/W2154113122","https://openalex.org/W2158025084","https://openalex.org/W2412861313","https://openalex.org/W2486441035","https://openalex.org/W4252981553","https://openalex.org/W4285719527"],"related_works":["https://openalex.org/W2074050424","https://openalex.org/W2184612063","https://openalex.org/W12495686","https://openalex.org/W2051264696","https://openalex.org/W175503027","https://openalex.org/W2098329690","https://openalex.org/W2116248904","https://openalex.org/W3132885237","https://openalex.org/W2160942127","https://openalex.org/W92192699"],"abstract_inverted_index":{"Knowledge-based":[0],"engineering":[1],"and":[2,14,144],"computational":[3],"intelligence":[4],"are":[5,31,56,82],"expected":[6],"to":[7,32,84,136,191],"become":[8],"core":[9],"technologies":[10],"in":[11,172],"the":[12,17,41,60,64,71,86,105,110,118,126,138,141,145],"design":[13],"manufacturing":[15],"for":[16,46,188],"next":[18],"generation":[19],"of":[20,43,51,67,88,90,109,125,147,152,157,159,161,182],"space":[21,108,128],"exploration":[22],"missions.":[23],"The":[24,49,150,180],"literature":[25],"is":[26,129,135],"contradictory":[27],"on":[28,155],"how":[29],"we":[30,97],"assess":[33],"such":[34],"systems.":[35,69],"Studies":[36],"indicate":[37,76,166],"significant":[38],"disagreement":[39],"regarding":[40],"amount":[42],"testing":[44,99,153,189,196],"needed":[45,83],"system":[47],"assessment.":[48],"sizes":[50],"standard":[52],"black-box":[53,61],"test":[54,116,119,148],"suites":[55],"impractically":[57],"large":[58],"since":[59],"approach":[62],"neglects":[63],"internal":[65,106],"structure":[66],"knowledge-based":[68,92,111],"On":[70],"contrary,":[72],"practical":[73],"results":[74,151],"repeatedly":[75],"that":[77,121,167],"only":[78],"a":[79,91,101,168],"few":[80],"tests":[81],"sample":[85],"range":[87],"behaviors":[89],"program.":[93],"In":[94],"this":[95],"paper,":[96],"model":[98],"as":[100,193,195],"search":[102,186,200],"process":[103],"over":[104],"state":[107,127],"system.":[112],"When":[113],"comparing":[114],"different":[115],"suites,":[117],"suite":[120],"examines":[122],"larger":[123],"portion":[124],"considered":[130],"more":[131,198],"complete.":[132],"Our":[133],"goal":[134],"investigate":[137],"trade-off":[139],"between":[140],"completeness":[142,173],"criterion":[143],"size":[146],"suites.":[149],"experiment":[154],"tens":[156],"thousands":[158],"mutants":[160],"real-world":[162],"knowledge":[163],"based":[164],"systems":[165],"very":[169],"limited":[170],"gain":[171],"can":[174],"be":[175,192],"achieved":[176],"through":[177],"prolonged":[178],"testing.":[179],"use":[181],"simple":[183],"(or":[184],"random)":[185],"strategies":[187],"appears":[190],"powerful":[194],"by":[197],"thorough":[199],"algorithms.":[201]},"counts_by_year":[{"year":2016,"cited_by_count":1},{"year":2014,"cited_by_count":1},{"year":2012,"cited_by_count":1}],"updated_date":"2026-05-21T06:26:12.895304","created_date":"2025-10-10T00:00:00"}