{"id":"https://openalex.org/W3211188136","doi":"https://doi.org/10.1145/3493700.3493704","title":"Data Synthesis for Testing Black-Box Machine Learning Models","display_name":"Data Synthesis for Testing Black-Box Machine Learning Models","publication_year":2022,"publication_date":"2022-01-07","ids":{"openalex":"https://openalex.org/W3211188136","doi":"https://doi.org/10.1145/3493700.3493704","mag":"3211188136"},"language":"en","primary_location":{"id":"doi:10.1145/3493700.3493704","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3493700.3493704","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 5th Joint International Conference on Data Science & Management of Data (9th ACM IKDD CODS and 27th COMAD)","raw_type":"proceedings-article"},"type":"preprint","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5010368277","display_name":"Diptikalyan Saha","orcid":"https://orcid.org/0000-0002-1583-5479"},"institutions":[{"id":"https://openalex.org/I4210103279","display_name":"IBM Research - India","ror":"https://ror.org/014wt7r80","country_code":"IN","type":"facility","lineage":["https://openalex.org/I1341412227","https://openalex.org/I4210103279","https://openalex.org/I4210114115"]}],"countries":["IN"],"is_corresponding":false,"raw_author_name":"Diptikalyan Saha","raw_affiliation_strings":["IBM Research India, IN"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM Research India, IN","institution_ids":["https://openalex.org/I4210103279"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5001896938","display_name":"Aniya Aggarwal","orcid":"https://orcid.org/0000-0001-8883-0030"},"institutions":[{"id":"https://openalex.org/I4210103279","display_name":"IBM Research - India","ror":"https://ror.org/014wt7r80","country_code":"IN","type":"facility","lineage":["https://openalex.org/I1341412227","https://openalex.org/I4210103279","https://openalex.org/I4210114115"]}],"countries":["IN"],"is_corresponding":false,"raw_author_name":"Aniya Aggarwal","raw_affiliation_strings":["IBM Research India, IN"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM Research India, IN","institution_ids":["https://openalex.org/I4210103279"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5039554087","display_name":"Sandeep Hans","orcid":"https://orcid.org/0000-0003-4986-0688"},"institutions":[{"id":"https://openalex.org/I4210103279","display_name":"IBM Research - India","ror":"https://ror.org/014wt7r80","country_code":"IN","type":"facility","lineage":["https://openalex.org/I1341412227","https://openalex.org/I4210103279","https://openalex.org/I4210114115"]}],"countries":["IN"],"is_corresponding":false,"raw_author_name":"Sandeep Hans","raw_affiliation_strings":["IBM Research India, IN"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM Research India, IN","institution_ids":["https://openalex.org/I4210103279"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.2775,"has_fulltext":false,"cited_by_count":2,"citation_normalized_percentile":{"value":0.60258228,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":94},"biblio":{"volume":null,"issue":null,"first_page":"110","last_page":"114"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.9970999956130981,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9918000102043152,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/black-box","display_name":"Black box","score":0.834363579750061},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7964775562286377},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.6671072244644165},{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.5968625545501709},{"id":"https://openalex.org/keywords/test-data","display_name":"Test data","score":0.5690338015556335},{"id":"https://openalex.org/keywords/data-modeling","display_name":"Data modeling","score":0.5523850321769714},{"id":"https://openalex.org/keywords/white-box-testing","display_name":"White-box testing","score":0.5286135077476501},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5265486836433411},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.5241339802742004},{"id":"https://openalex.org/keywords/training-set","display_name":"Training set","score":0.47290122509002686},{"id":"https://openalex.org/keywords/test","display_name":"Test (biology)","score":0.47034695744514465},{"id":"https://openalex.org/keywords/data-set","display_name":"Data set","score":0.46539032459259033},{"id":"https://openalex.org/keywords/test-set","display_name":"Test set","score":0.4472721517086029},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.4333675801753998},{"id":"https://openalex.org/keywords/model-based-testing","display_name":"Model-based testing","score":0.4144699275493622},{"id":"https://openalex.org/keywords/test-case","display_name":"Test case","score":0.2680966258049011},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.13666626811027527},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.10408169031143188},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.09183627367019653}],"concepts":[{"id":"https://openalex.org/C94966114","wikidata":"https://www.wikidata.org/wiki/Q29256","display_name":"Black box","level":2,"score":0.834363579750061},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7964775562286377},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.6671072244644165},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.5968625545501709},{"id":"https://openalex.org/C16910744","wikidata":"https://www.wikidata.org/wiki/Q7705759","display_name":"Test data","level":2,"score":0.5690338015556335},{"id":"https://openalex.org/C67186912","wikidata":"https://www.wikidata.org/wiki/Q367664","display_name":"Data modeling","level":2,"score":0.5523850321769714},{"id":"https://openalex.org/C162443782","wikidata":"https://www.wikidata.org/wiki/Q1066228","display_name":"White-box testing","level":5,"score":0.5286135077476501},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5265486836433411},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.5241339802742004},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.47290122509002686},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.47034695744514465},{"id":"https://openalex.org/C58489278","wikidata":"https://www.wikidata.org/wiki/Q1172284","display_name":"Data set","level":2,"score":0.46539032459259033},{"id":"https://openalex.org/C169903167","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Test set","level":2,"score":0.4472721517086029},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.4333675801753998},{"id":"https://openalex.org/C165825675","wikidata":"https://www.wikidata.org/wiki/Q1399743","display_name":"Model-based testing","level":4,"score":0.4144699275493622},{"id":"https://openalex.org/C128942645","wikidata":"https://www.wikidata.org/wiki/Q1568346","display_name":"Test case","level":3,"score":0.2680966258049011},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.13666626811027527},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.10408169031143188},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.09183627367019653},{"id":"https://openalex.org/C152877465","wikidata":"https://www.wikidata.org/wiki/Q208042","display_name":"Regression analysis","level":2,"score":0.0},{"id":"https://openalex.org/C163258240","wikidata":"https://www.wikidata.org/wiki/Q25342","display_name":"Power (physics)","level":2,"score":0.0},{"id":"https://openalex.org/C186846655","wikidata":"https://www.wikidata.org/wiki/Q3398377","display_name":"Software construction","level":4,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C149091818","wikidata":"https://www.wikidata.org/wiki/Q2429814","display_name":"Software system","level":3,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3493700.3493704","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3493700.3493704","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 5th Joint International Conference on Data Science & Management of Data (9th ACM IKDD CODS and 27th COMAD)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":35,"referenced_works":["https://openalex.org/W1959608418","https://openalex.org/W2014352947","https://openalex.org/W2031648200","https://openalex.org/W2041650849","https://openalex.org/W2051317694","https://openalex.org/W2099471712","https://openalex.org/W2129281431","https://openalex.org/W2134682899","https://openalex.org/W2146950091","https://openalex.org/W2148143831","https://openalex.org/W2158406371","https://openalex.org/W2166481425","https://openalex.org/W2182722412","https://openalex.org/W2282821441","https://openalex.org/W2616028256","https://openalex.org/W2730550703","https://openalex.org/W2753845591","https://openalex.org/W2785011159","https://openalex.org/W2804337238","https://openalex.org/W2809701591","https://openalex.org/W2922015121","https://openalex.org/W2945295328","https://openalex.org/W2954629067","https://openalex.org/W2963165448","https://openalex.org/W2967682612","https://openalex.org/W2968940383","https://openalex.org/W2970533824","https://openalex.org/W3008021358","https://openalex.org/W3035507081","https://openalex.org/W3099444373","https://openalex.org/W3103145119","https://openalex.org/W3104149808","https://openalex.org/W3120740533","https://openalex.org/W3183522126","https://openalex.org/W4255375128"],"related_works":["https://openalex.org/W1501213200","https://openalex.org/W112492789","https://openalex.org/W1532322220","https://openalex.org/W2069818478","https://openalex.org/W2131714870","https://openalex.org/W4242844989","https://openalex.org/W2171640008","https://openalex.org/W4319452366","https://openalex.org/W2543388663","https://openalex.org/W2041668232"],"abstract_inverted_index":{"The":[0,16],"increasing":[1],"usage":[2],"of":[3,10,13,19,49,64,79],"machine":[4,71],"learning":[5,72],"models":[6],"raises":[7],"the":[8,11,77],"question":[9],"reliability":[12],"these":[14],"models.":[15,43,73],"current":[17],"practice":[18],"testing":[20],"with":[21,54],"limited":[22],"data":[23,37,53],"is":[24],"often":[25],"insufficient.":[26],"In":[27],"this":[28],"paper,":[29],"we":[30],"provide":[31],"a":[32,61],"framework":[33],"for":[34],"automated":[35],"test":[36,40,60],"synthesis":[38],"to":[39,59,67],"black-box":[41],"ML/DL":[42],"We":[44,74],"address":[45],"an":[46],"important":[47],"challenge":[48],"generating":[50],"realistic":[51],"user-controllable":[52],"model":[55],"agnostic":[56],"coverage":[57],"criteria":[58],"varied":[62],"set":[63],"properties,":[65],"essentially":[66],"increase":[68],"trust":[69],"in":[70],"experimentally":[75],"demonstrate":[76],"effectiveness":[78],"our":[80],"technique.":[81]},"counts_by_year":[{"year":2024,"cited_by_count":1},{"year":2022,"cited_by_count":1}],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-10-10T00:00:00"}