{"id":"https://openalex.org/W4283073501","doi":"https://doi.org/10.1109/icse-seip55303.2022.9793981","title":"Testing Machine Learning Systems in Industry: An Empirical Study","display_name":"Testing Machine Learning Systems in Industry: An Empirical Study","publication_year":2022,"publication_date":"2022-05-01","ids":{"openalex":"https://openalex.org/W4283073501","doi":"https://doi.org/10.1109/icse-seip55303.2022.9793981"},"language":"en","primary_location":{"id":"doi:10.1109/icse-seip55303.2022.9793981","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icse-seip55303.2022.9793981","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2022 IEEE/ACM 44th International Conference on Software Engineering: Software Engineering in Practice (ICSE-SEIP)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5028327074","display_name":"Shuyue Li","orcid":"https://orcid.org/0000-0001-6009-1707"},"institutions":[{"id":"https://openalex.org/I87445476","display_name":"Xi'an Jiaotong University","ror":"https://ror.org/017zhmm22","country_code":"CN","type":"education","lineage":["https://openalex.org/I87445476"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Shuyue Li","raw_affiliation_strings":["Xi'an Jiaotong University,Xi'an, Shaanxi,China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Xi'an Jiaotong University,Xi'an, Shaanxi,China","institution_ids":["https://openalex.org/I87445476"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5032965724","display_name":"Jiaqi Guo","orcid":"https://orcid.org/0000-0001-6340-3331"},"institutions":[{"id":"https://openalex.org/I87445476","display_name":"Xi'an Jiaotong University","ror":"https://ror.org/017zhmm22","country_code":"CN","type":"education","lineage":["https://openalex.org/I87445476"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jiaqi Guo","raw_affiliation_strings":["Xi'an Jiaotong University,Xi'an, Shaanxi,China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Xi'an Jiaotong University,Xi'an, Shaanxi,China","institution_ids":["https://openalex.org/I87445476"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5025118710","display_name":"Jian\u2013Guang Lou","orcid":null},"institutions":[{"id":"https://openalex.org/I87445476","display_name":"Xi'an Jiaotong University","ror":"https://ror.org/017zhmm22","country_code":"CN","type":"education","lineage":["https://openalex.org/I87445476"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jian-Guang Lou","raw_affiliation_strings":["Xi'an Jiaotong University,Xi'an, Shaanxi,China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Xi'an Jiaotong University,Xi'an, Shaanxi,China","institution_ids":["https://openalex.org/I87445476"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100692958","display_name":"Ming Fan","orcid":"https://orcid.org/0000-0002-9327-0987"},"institutions":[{"id":"https://openalex.org/I87445476","display_name":"Xi'an Jiaotong University","ror":"https://ror.org/017zhmm22","country_code":"CN","type":"education","lineage":["https://openalex.org/I87445476"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Ming Fan","raw_affiliation_strings":["Xi'an Jiaotong University,Xi'an, Shaanxi,China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Xi'an Jiaotong University,Xi'an, Shaanxi,China","institution_ids":["https://openalex.org/I87445476"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101554099","display_name":"Ting Liu","orcid":"https://orcid.org/0000-0002-7600-0934"},"institutions":[{"id":"https://openalex.org/I87445476","display_name":"Xi'an Jiaotong University","ror":"https://ror.org/017zhmm22","country_code":"CN","type":"education","lineage":["https://openalex.org/I87445476"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Ting Liu","raw_affiliation_strings":["Xi'an Jiaotong University,Xi'an, Shaanxi,China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Xi'an Jiaotong University,Xi'an, Shaanxi,China","institution_ids":["https://openalex.org/I87445476"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100331488","display_name":"Dongmei Zhang","orcid":"https://orcid.org/0000-0002-9230-2799"},"institutions":[{"id":"https://openalex.org/I87445476","display_name":"Xi'an Jiaotong University","ror":"https://ror.org/017zhmm22","country_code":"CN","type":"education","lineage":["https://openalex.org/I87445476"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Dongmei Zhang","raw_affiliation_strings":["Xi'an Jiaotong University,Xi'an, Shaanxi,China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Xi'an Jiaotong University,Xi'an, Shaanxi,China","institution_ids":["https://openalex.org/I87445476"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I87445476"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":4,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"263","last_page":"272"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.9976999759674072,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9908000230789185,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6611112356185913},{"id":"https://openalex.org/keywords/empirical-research","display_name":"Empirical research","score":0.4467596411705017},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.3833921551704407},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.35258767008781433}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6611112356185913},{"id":"https://openalex.org/C120936955","wikidata":"https://www.wikidata.org/wiki/Q2155640","display_name":"Empirical research","level":2,"score":0.4467596411705017},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3833921551704407},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.35258767008781433},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/icse-seip55303.2022.9793981","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icse-seip55303.2022.9793981","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2022 IEEE/ACM 44th International Conference on Software Engineering: Software Engineering in Practice (ICSE-SEIP)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/9","score":0.6200000047683716,"display_name":"Industry, innovation and infrastructure"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":41,"referenced_works":["https://openalex.org/W171406816","https://openalex.org/W179875071","https://openalex.org/W596281310","https://openalex.org/W953560957","https://openalex.org/W1524053243","https://openalex.org/W1682403713","https://openalex.org/W1965555277","https://openalex.org/W2008620926","https://openalex.org/W2120615054","https://openalex.org/W2189162242","https://openalex.org/W2306526421","https://openalex.org/W2352511489","https://openalex.org/W2552408584","https://openalex.org/W2744999500","https://openalex.org/W2753845591","https://openalex.org/W2782864149","https://openalex.org/W2891728317","https://openalex.org/W2903158431","https://openalex.org/W2922234936","https://openalex.org/W2944221325","https://openalex.org/W2957905354","https://openalex.org/W2965189679","https://openalex.org/W2968940383","https://openalex.org/W2970433196","https://openalex.org/W3007157104","https://openalex.org/W3023533951","https://openalex.org/W3035507081","https://openalex.org/W3099444373","https://openalex.org/W3100935330","https://openalex.org/W3104217200","https://openalex.org/W3178258123","https://openalex.org/W4288301650","https://openalex.org/W4294349862","https://openalex.org/W4385245566","https://openalex.org/W6687241523","https://openalex.org/W6711870810","https://openalex.org/W6744110554","https://openalex.org/W6765411980","https://openalex.org/W6767260480","https://openalex.org/W6785667435","https://openalex.org/W6910318977"],"related_works":["https://openalex.org/W2961085424","https://openalex.org/W4306674287","https://openalex.org/W3046775127","https://openalex.org/W3107602296","https://openalex.org/W4394896187","https://openalex.org/W3170094116","https://openalex.org/W4386462264","https://openalex.org/W4364306694","https://openalex.org/W4312192474","https://openalex.org/W4283697347"],"abstract_inverted_index":{"Machine":[0],"learning":[1],"becomes":[2],"increasingly":[3],"prevalent":[4],"and":[5,82,109,113,118,140,142,162,179,199],"integrated":[6],"into":[7],"a":[8,77],"wide":[9],"range":[10],"of":[11,124],"software":[12],"systems.":[13],"These":[14],"systems,":[15,18,42],"named":[16],"ML":[17,41,54,69,87,137,151],"must":[19],"be":[20],"adequately":[21],"tested":[22],"to":[23,37],"gain":[24],"confidence":[25],"that":[26],"they":[27],"behave":[28],"correctly.":[29],"Although":[30],"many":[31],"research":[32,197],"efforts":[33],"have":[34],"been":[35],"devoted":[36],"testing":[38,52,101],"technologies":[39],"for":[40,204],"the":[43,53,63,66,115,122,125,160,163,196],"industrial":[44,97],"teams":[45],"are":[46],"faced":[47,143],"with":[48,79,84,144,182],"new":[49],"challenges":[50,120],"on":[51,65,99,136,165,173,187,191],"systems":[55,152],"in":[56,68,133,150],"real-world":[57],"settings.":[58],"To":[59],"absorb":[60],"inspirations":[61],"from":[62,89,121,154],"industry":[64],"problems":[67],"testing,":[70],"we":[71,194],"conducted":[72,132],"an":[73],"empirical":[74],"study":[75,94],"including":[76],"survey":[78],"87":[80],"responses":[81],"interviews":[83],"7":[85],"senior":[86],"practitioners":[88],"well-known":[90],"IT":[91],"companies.":[92],"Our":[93],"uncovers":[95],"significant":[96],"concerns":[98],"major":[100,156],"activities,":[102],"i.e.,":[103],"test":[104,107,110],"data":[105,129],"collection,":[106],"execution,":[108],"result":[111,170],"analysis,":[112],"also":[114,200],"good":[116],"practices":[117],"open":[119],"perspective":[123],"industry.":[126],"(1)":[127],"Test":[128,148,169],"collection":[130],"is":[131,180],"different":[134,145],"ways":[135],"model,":[138],"data,":[139],"code":[141],"challenges.":[146],"(2)":[147],"execution":[149],"suffers":[153],"two":[155],"problems:":[157],"entanglement":[158],"among":[159],"components":[161],"regression":[164],"model":[166],"performance.":[167],"(3)":[168],"analysis":[171],"centers":[172],"quantitative":[174],"methods,":[175],"e.g.,":[176],"metric-based":[177],"evaluation,":[178],"combined":[181],"some":[183,202],"qualitative":[184],"methods":[185],"based":[186],"practitioners\u2019":[188],"experience.":[189],"Based":[190],"our":[192],"findings,":[193],"highlight":[195],"opportunities":[198],"provide":[201],"implications":[203],"practitioners.":[205]},"counts_by_year":[{"year":2024,"cited_by_count":2},{"year":2023,"cited_by_count":2}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}