{"id":"https://openalex.org/W2969466556","doi":"https://doi.org/10.1109/access.2019.2937107","title":"Testing and Quality Validation for AI Software\u2013Perspectives, Issues, and Practices","display_name":"Testing and Quality Validation for AI Software\u2013Perspectives, Issues, and Practices","publication_year":2019,"publication_date":"2019-01-01","ids":{"openalex":"https://openalex.org/W2969466556","doi":"https://doi.org/10.1109/access.2019.2937107","mag":"2969466556"},"language":"en","primary_location":{"id":"doi:10.1109/access.2019.2937107","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2019.2937107","pdf_url":"https://ieeexplore.ieee.org/ielx7/6287639/8600701/08811507.pdf","source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://ieeexplore.ieee.org/ielx7/6287639/8600701/08811507.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5002015838","display_name":"Chuanqi Tao","orcid":"https://orcid.org/0000-0002-0698-7307"},"institutions":[{"id":"https://openalex.org/I881766915","display_name":"Nanjing University","ror":"https://ror.org/01rxvg760","country_code":"CN","type":"education","lineage":["https://openalex.org/I881766915"]},{"id":"https://openalex.org/I9842412","display_name":"Nanjing University of Aeronautics and Astronautics","ror":"https://ror.org/01scyh794","country_code":"CN","type":"education","lineage":["https://openalex.org/I9842412"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Chuanqi Tao","raw_affiliation_strings":["College of Computer Science and Technology, Nanjing University of Aeronautics and Astronautics, Nanjing, China","Ministry Key Laboratory for Safety-Critical Software Development and Verification, Nanjing University of Aeronautics and Astronautics, Nanjing, China","State Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, China"],"affiliations":[{"raw_affiliation_string":"College of Computer Science and Technology, Nanjing University of Aeronautics and Astronautics, Nanjing, China","institution_ids":["https://openalex.org/I9842412"]},{"raw_affiliation_string":"Ministry Key Laboratory for Safety-Critical Software Development and Verification, Nanjing University of Aeronautics and Astronautics, Nanjing, China","institution_ids":["https://openalex.org/I9842412"]},{"raw_affiliation_string":"State Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, China","institution_ids":["https://openalex.org/I881766915"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5074631753","display_name":"Jerry Gao","orcid":"https://orcid.org/0000-0002-1051-5839"},"institutions":[{"id":"https://openalex.org/I51504820","display_name":"San Jose State University","ror":"https://ror.org/04qyvz380","country_code":"US","type":"education","lineage":["https://openalex.org/I51504820"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jerry Gao","raw_affiliation_strings":["Department of Computer Engineering, San Jos\u00e9 State University, San Jose, USA"],"affiliations":[{"raw_affiliation_string":"Department of Computer Engineering, San Jos\u00e9 State University, San Jose, USA","institution_ids":["https://openalex.org/I51504820"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5045242853","display_name":"Tiexin Wang","orcid":"https://orcid.org/0000-0002-5432-3812"},"institutions":[{"id":"https://openalex.org/I9842412","display_name":"Nanjing University of Aeronautics and Astronautics","ror":"https://ror.org/01scyh794","country_code":"CN","type":"education","lineage":["https://openalex.org/I9842412"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Tiexin Wang","raw_affiliation_strings":["College of Computer Science and Technology, Nanjing University of Aeronautics and Astronautics, Nanjing, China","Ministry Key Laboratory for Safety-Critical Software Development and Verification, Nanjing University of Aeronautics and Astronautics, Nanjing, China"],"affiliations":[{"raw_affiliation_string":"College of Computer Science and Technology, Nanjing University of Aeronautics and Astronautics, Nanjing, China","institution_ids":["https://openalex.org/I9842412"]},{"raw_affiliation_string":"Ministry Key Laboratory for Safety-Critical Software Development and Verification, Nanjing University of Aeronautics and Astronautics, Nanjing, China","institution_ids":["https://openalex.org/I9842412"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5002015838"],"corresponding_institution_ids":["https://openalex.org/I881766915","https://openalex.org/I9842412"],"apc_list":{"value":1850,"currency":"USD","value_usd":1850},"apc_paid":{"value":1850,"currency":"USD","value_usd":1850},"fwci":6.0198,"has_fulltext":true,"cited_by_count":56,"citation_normalized_percentile":{"value":0.96339924,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":100},"biblio":{"volume":"7","issue":null,"first_page":"120164","last_page":"120175"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9947999715805054,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9947999715805054,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11512","display_name":"Anomaly Detection Techniques and Applications","score":0.9936000108718872,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9929999709129333,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8291463255882263},{"id":"https://openalex.org/keywords/software-quality","display_name":"Software quality","score":0.6673814058303833},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.543609619140625},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.4993400573730469},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.4734589457511902},{"id":"https://openalex.org/keywords/verification-and-validation","display_name":"Verification and validation","score":0.4631440341472626},{"id":"https://openalex.org/keywords/software-reliability-testing","display_name":"Software reliability testing","score":0.4595550298690796},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.43566563725471497},{"id":"https://openalex.org/keywords/software-quality-control","display_name":"Software quality control","score":0.4186943769454956},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.38336148858070374},{"id":"https://openalex.org/keywords/software-development","display_name":"Software development","score":0.28587305545806885},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.08125215768814087}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8291463255882263},{"id":"https://openalex.org/C117447612","wikidata":"https://www.wikidata.org/wiki/Q1412670","display_name":"Software quality","level":4,"score":0.6673814058303833},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.543609619140625},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.4993400573730469},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.4734589457511902},{"id":"https://openalex.org/C48002344","wikidata":"https://www.wikidata.org/wiki/Q2919644","display_name":"Verification and validation","level":2,"score":0.4631440341472626},{"id":"https://openalex.org/C52928878","wikidata":"https://www.wikidata.org/wiki/Q7554226","display_name":"Software reliability testing","level":5,"score":0.4595550298690796},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.43566563725471497},{"id":"https://openalex.org/C176035894","wikidata":"https://www.wikidata.org/wiki/Q7554350","display_name":"Software quality control","level":5,"score":0.4186943769454956},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.38336148858070374},{"id":"https://openalex.org/C529173508","wikidata":"https://www.wikidata.org/wiki/Q638608","display_name":"Software development","level":3,"score":0.28587305545806885},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.08125215768814087},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C21547014","wikidata":"https://www.wikidata.org/wiki/Q1423657","display_name":"Operations management","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/access.2019.2937107","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2019.2937107","pdf_url":"https://ieeexplore.ieee.org/ielx7/6287639/8600701/08811507.pdf","source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},{"id":"pmh:oai:doaj.org/article:7d9aa360f1004193b92858edc4aa2310","is_oa":true,"landing_page_url":"https://doaj.org/article/7d9aa360f1004193b92858edc4aa2310","pdf_url":null,"source":{"id":"https://openalex.org/S112646816","display_name":"SHILAP Revista de lepidopterolog\u00eda","issn_l":"0300-5267","issn":["0300-5267","2340-4078"],"is_oa":true,"is_in_doaj":true,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"IEEE Access, Vol 7, Pp 120164-120175 (2019)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.1109/access.2019.2937107","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2019.2937107","pdf_url":"https://ieeexplore.ieee.org/ielx7/6287639/8600701/08811507.pdf","source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G1121271761","display_name":null,"funder_award_id":"Program","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G1231421488","display_name":null,"funder_award_id":"under","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G1533970194","display_name":null,"funder_award_id":"61402229","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G1652543168","display_name":null,"funder_award_id":"2018YFB1003900","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G1958249691","display_name":null,"funder_award_id":"2018YFB1003900","funder_id":"https://openalex.org/F4320335777","funder_display_name":"National Key Research and Development Program of China"},{"id":"https://openalex.org/G2087396116","display_name":null,"funder_award_id":"China","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G2376276132","display_name":null,"funder_award_id":"China","funder_id":"https://openalex.org/F4320335787","funder_display_name":"Fundamental Research Funds for the Central Universities"},{"id":"https://openalex.org/G3317480652","display_name":null,"funder_award_id":"Science","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G3435950671","display_name":null,"funder_award_id":"and Gr","funder_id":"https://openalex.org/F4320335777","funder_display_name":"National Key Research and Development Program of China"},{"id":"https://openalex.org/G391238517","display_name":null,"funder_award_id":", and","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G4713543433","display_name":null,"funder_award_id":"61402","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G5845821273","display_name":null,"funder_award_id":"NS2019058","funder_id":"https://openalex.org/F4320335787","funder_display_name":"Fundamental Research Funds for the Central Universities"},{"id":"https://openalex.org/G5939423041","display_name":null,"funder_award_id":"Technology","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G5994120800","display_name":null,"funder_award_id":"Natural","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G6529085909","display_name":null,"funder_award_id":"KFKT2018","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G7114947763","display_name":null,"funder_award_id":"KFKT2018B19","funder_id":"https://openalex.org/F4320326895","funder_display_name":"State Key Laboratory of Novel Software Technology"},{"id":"https://openalex.org/G7822769749","display_name":null,"funder_award_id":"2019058","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G7844191210","display_name":null,"funder_award_id":"61602267","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G8951484681","display_name":null,"funder_award_id":"Grant","funder_id":"https://openalex.org/F4320335787","funder_display_name":"Fundamental Research Funds for the Central Universities"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320326895","display_name":"State Key Laboratory of Novel Software Technology","ror":null},{"id":"https://openalex.org/F4320335777","display_name":"National Key Research and Development Program of China","ror":null},{"id":"https://openalex.org/F4320335787","display_name":"Fundamental Research Funds for the Central Universities","ror":null}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2969466556.pdf","grobid_xml":"https://content.openalex.org/works/W2969466556.grobid-xml"},"referenced_works_count":50,"referenced_works":["https://openalex.org/W12677450","https://openalex.org/W650635229","https://openalex.org/W1519683776","https://openalex.org/W1550282031","https://openalex.org/W1564605495","https://openalex.org/W1648383461","https://openalex.org/W1963918008","https://openalex.org/W1992639336","https://openalex.org/W2001771035","https://openalex.org/W2041650849","https://openalex.org/W2060291526","https://openalex.org/W2060993269","https://openalex.org/W2077091911","https://openalex.org/W2091093620","https://openalex.org/W2096956640","https://openalex.org/W2106953271","https://openalex.org/W2134124281","https://openalex.org/W2153344230","https://openalex.org/W2157225187","https://openalex.org/W2171728358","https://openalex.org/W2173891962","https://openalex.org/W2309401901","https://openalex.org/W2321262777","https://openalex.org/W2417764354","https://openalex.org/W2438196771","https://openalex.org/W2512953987","https://openalex.org/W2521921275","https://openalex.org/W2626124486","https://openalex.org/W2674867493","https://openalex.org/W2722632382","https://openalex.org/W2734479159","https://openalex.org/W2760805760","https://openalex.org/W2763314014","https://openalex.org/W2789444109","https://openalex.org/W2789860113","https://openalex.org/W2797143217","https://openalex.org/W2804252870","https://openalex.org/W2804337238","https://openalex.org/W2810605094","https://openalex.org/W2889217990","https://openalex.org/W2891182845","https://openalex.org/W2897510551","https://openalex.org/W2906628499","https://openalex.org/W2927980542","https://openalex.org/W2933980179","https://openalex.org/W3099444373","https://openalex.org/W4234789351","https://openalex.org/W4237158495","https://openalex.org/W6600539062","https://openalex.org/W6631148570"],"related_works":["https://openalex.org/W2549211027","https://openalex.org/W234065253","https://openalex.org/W1981645631","https://openalex.org/W1580492598","https://openalex.org/W2090249529","https://openalex.org/W2117398427","https://openalex.org/W2406746345","https://openalex.org/W4238386252","https://openalex.org/W2184255123","https://openalex.org/W2798306226"],"abstract_inverted_index":{"With":[0],"the":[1,152,157],"fast":[2],"growth":[3],"of":[4,97,156],"artificial":[5],"intelligence":[6],"and":[7,13,26,31,65,75,104,115,125,154],"big":[8],"data":[9],"computing":[10],"technologies,":[11],"more":[12,14],"software":[15,89,99,110],"service":[16],"systems":[17],"have":[18],"been":[19],"developed":[20],"using":[21],"diverse":[22],"machine":[23],"learning":[24],"models":[25],"technologies":[27],"to":[28,39,72],"make":[29],"business":[30],"intelligent":[32,41],"decisions":[33],"based":[34],"on":[35,84,134],"their":[36],"multimedia":[37],"input":[38],"achieve":[40],"features,":[42],"such":[43],"as":[44],"image":[45,139],"recognition,":[46],"recommendation,":[47],"decision":[48],"making,":[49],"prediction,":[50],"etc.":[51],"Nevertheless,":[52],"there":[53],"are":[54,113,119,128],"increasing":[55],"quality":[56,76,85,123,135],"problems":[57],"resulting":[58],"in":[59,63],"erroneous":[60],"testing":[61,74,100,111,117,147],"costs":[62],"enterprises":[64],"businesses.":[66],"Existing":[67],"work":[68],"seldom":[69],"discusses":[70],"how":[71],"perform":[73],"validation":[77,86,136],"for":[78,87,101,137],"AI":[79,88,98,109],"software.":[80],"This":[81],"paper":[82,93],"focuses":[83],"function":[90],"features.":[91],"The":[92],"provides":[94],"our":[95],"understanding":[96],"new":[102],"features":[103],"requirements.":[105],"In":[106],"addition,":[107],"current":[108],"categories":[112],"presented":[114],"different":[116],"approaches":[118],"discussed.":[120],"Moreover,":[121],"test":[122],"assessment":[124],"criteria":[126],"analysis":[127],"illustrated.":[129],"Furthermore,":[130],"a":[131,145],"practical":[132],"study":[133],"an":[138],"recognition":[140],"system":[141],"is":[142],"performed":[143],"through":[144],"metamorphic":[146],"method.":[148],"Study":[149],"results":[150],"show":[151],"feasibility":[153],"effectiveness":[155],"approach.":[158]},"counts_by_year":[{"year":2026,"cited_by_count":5},{"year":2025,"cited_by_count":8},{"year":2024,"cited_by_count":13},{"year":2023,"cited_by_count":12},{"year":2022,"cited_by_count":7},{"year":2021,"cited_by_count":7},{"year":2020,"cited_by_count":3},{"year":2019,"cited_by_count":1}],"updated_date":"2026-03-18T14:38:29.013473","created_date":"2025-10-10T00:00:00"}