{"id":"https://openalex.org/W4284669642","doi":"https://doi.org/10.1145/3510003.3510068","title":"Automated testing of software that uses machine learning APIs","display_name":"Automated testing of software that uses machine learning APIs","publication_year":2022,"publication_date":"2022-05-21","ids":{"openalex":"https://openalex.org/W4284669642","doi":"https://doi.org/10.1145/3510003.3510068"},"language":"en","primary_location":{"id":"doi:10.1145/3510003.3510068","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3510003.3510068","pdf_url":null,"source":{"id":"https://openalex.org/S4363608872","display_name":"Proceedings of the 44th International Conference on Software Engineering","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 44th International Conference on Software Engineering","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5016446062","display_name":"Chengcheng Wan","orcid":"https://orcid.org/0000-0001-9162-9688"},"institutions":[{"id":"https://openalex.org/I40347166","display_name":"University of Chicago","ror":"https://ror.org/024mw5h28","country_code":"US","type":"education","lineage":["https://openalex.org/I40347166"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Chengcheng Wan","raw_affiliation_strings":["University of Chicago"],"affiliations":[{"raw_affiliation_string":"University of Chicago","institution_ids":["https://openalex.org/I40347166"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5018640318","display_name":"Shicheng Liu","orcid":"https://orcid.org/0000-0002-0843-284X"},"institutions":[{"id":"https://openalex.org/I40347166","display_name":"University of Chicago","ror":"https://ror.org/024mw5h28","country_code":"US","type":"education","lineage":["https://openalex.org/I40347166"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Shicheng Liu","raw_affiliation_strings":["University of Chicago"],"affiliations":[{"raw_affiliation_string":"University of Chicago","institution_ids":["https://openalex.org/I40347166"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5060048530","display_name":"Sophie Xie","orcid":"https://orcid.org/0009-0000-5484-5234"},"institutions":[{"id":"https://openalex.org/I2801525552","display_name":"Whitney Museum of American Art","ror":"https://ror.org/01g1t6g78","country_code":"US","type":"archive","lineage":["https://openalex.org/I2801525552"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Sophie Xie","raw_affiliation_strings":["Whitney Young High School"],"affiliations":[{"raw_affiliation_string":"Whitney Young High School","institution_ids":["https://openalex.org/I2801525552"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100376172","display_name":"Yifan Liu","orcid":"https://orcid.org/0009-0008-0229-132X"},"institutions":[{"id":"https://openalex.org/I40347166","display_name":"University of Chicago","ror":"https://ror.org/024mw5h28","country_code":"US","type":"education","lineage":["https://openalex.org/I40347166"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yifan Liu","raw_affiliation_strings":["University of Chicago"],"affiliations":[{"raw_affiliation_string":"University of Chicago","institution_ids":["https://openalex.org/I40347166"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5080833704","display_name":"Henry Hoffmann","orcid":"https://orcid.org/0000-0003-0816-8150"},"institutions":[{"id":"https://openalex.org/I40347166","display_name":"University of Chicago","ror":"https://ror.org/024mw5h28","country_code":"US","type":"education","lineage":["https://openalex.org/I40347166"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Henry Hoffmann","raw_affiliation_strings":["University of Chicago"],"affiliations":[{"raw_affiliation_string":"University of Chicago","institution_ids":["https://openalex.org/I40347166"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5001371942","display_name":"Michael Maire","orcid":"https://orcid.org/0000-0002-9778-6673"},"institutions":[{"id":"https://openalex.org/I40347166","display_name":"University of Chicago","ror":"https://ror.org/024mw5h28","country_code":"US","type":"education","lineage":["https://openalex.org/I40347166"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Michael Maire","raw_affiliation_strings":["University of Chicago"],"affiliations":[{"raw_affiliation_string":"University of Chicago","institution_ids":["https://openalex.org/I40347166"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5061448314","display_name":"Shan Lu","orcid":"https://orcid.org/0000-0002-0757-4600"},"institutions":[{"id":"https://openalex.org/I40347166","display_name":"University of Chicago","ror":"https://ror.org/024mw5h28","country_code":"US","type":"education","lineage":["https://openalex.org/I40347166"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Shan Lu","raw_affiliation_strings":["University of Chicago"],"affiliations":[{"raw_affiliation_string":"University of Chicago","institution_ids":["https://openalex.org/I40347166"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":7,"corresponding_author_ids":["https://openalex.org/A5016446062"],"corresponding_institution_ids":["https://openalex.org/I40347166"],"apc_list":null,"apc_paid":null,"fwci":1.8709,"has_fulltext":false,"cited_by_count":19,"citation_normalized_percentile":{"value":0.87610429,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"212","last_page":"224"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11512","display_name":"Anomaly Detection Techniques and Applications","score":0.9958000183105469,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.991599977016449,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7912867069244385},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.5997987985610962},{"id":"https://openalex.org/keywords/software-construction","display_name":"Software construction","score":0.45248228311538696},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.42661458253860474},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.4129481017589569},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.37371575832366943},{"id":"https://openalex.org/keywords/software-system","display_name":"Software system","score":0.35041552782058716},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.33448266983032227},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.2512136697769165}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7912867069244385},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.5997987985610962},{"id":"https://openalex.org/C186846655","wikidata":"https://www.wikidata.org/wiki/Q3398377","display_name":"Software construction","level":4,"score":0.45248228311538696},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.42661458253860474},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.4129481017589569},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.37371575832366943},{"id":"https://openalex.org/C149091818","wikidata":"https://www.wikidata.org/wiki/Q2429814","display_name":"Software system","level":3,"score":0.35041552782058716},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.33448266983032227},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.2512136697769165}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3510003.3510068","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3510003.3510068","pdf_url":null,"source":{"id":"https://openalex.org/S4363608872","display_name":"Proceedings of the 44th International Conference on Software Engineering","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 44th International Conference on Software Engineering","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/16","score":0.46000000834465027,"display_name":"Peace, Justice and strong institutions"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":73,"referenced_works":["https://openalex.org/W1728842521","https://openalex.org/W2101234009","https://openalex.org/W2113730959","https://openalex.org/W2157309873","https://openalex.org/W2176208359","https://openalex.org/W2182687482","https://openalex.org/W2332506150","https://openalex.org/W2405754747","https://openalex.org/W2490192148","https://openalex.org/W2518851919","https://openalex.org/W2522956255","https://openalex.org/W2552408584","https://openalex.org/W2600531987","https://openalex.org/W2616028256","https://openalex.org/W2730550703","https://openalex.org/W2732493711","https://openalex.org/W2739781989","https://openalex.org/W2750930819","https://openalex.org/W2757656223","https://openalex.org/W2768281325","https://openalex.org/W2775678306","https://openalex.org/W2799640043","https://openalex.org/W2859484040","https://openalex.org/W2885205917","https://openalex.org/W2887009404","https://openalex.org/W2887649345","https://openalex.org/W2888307014","https://openalex.org/W2888483524","https://openalex.org/W2898851569","https://openalex.org/W2898868990","https://openalex.org/W2901595218","https://openalex.org/W2922234936","https://openalex.org/W2946581789","https://openalex.org/W2949346385","https://openalex.org/W2954903132","https://openalex.org/W2957905354","https://openalex.org/W2963327228","https://openalex.org/W2963417953","https://openalex.org/W2977524784","https://openalex.org/W2978132101","https://openalex.org/W2982277431","https://openalex.org/W2991533980","https://openalex.org/W2994987245","https://openalex.org/W2998795119","https://openalex.org/W2999072931","https://openalex.org/W2999193264","https://openalex.org/W2999450509","https://openalex.org/W2999691867","https://openalex.org/W3002944878","https://openalex.org/W3004706101","https://openalex.org/W3020029582","https://openalex.org/W3033191194","https://openalex.org/W3034750315","https://openalex.org/W3035671939","https://openalex.org/W3039960847","https://openalex.org/W3042650574","https://openalex.org/W3043238989","https://openalex.org/W3043716470","https://openalex.org/W3049756761","https://openalex.org/W3091633490","https://openalex.org/W3104896058","https://openalex.org/W3105249702","https://openalex.org/W3105599650","https://openalex.org/W3109004940","https://openalex.org/W3123045479","https://openalex.org/W3125205424","https://openalex.org/W6670247265","https://openalex.org/W6676297131","https://openalex.org/W6756021500","https://openalex.org/W6775876232","https://openalex.org/W6780887550","https://openalex.org/W6790978476","https://openalex.org/W6795336002"],"related_works":["https://openalex.org/W2049696050","https://openalex.org/W2142884843","https://openalex.org/W1510278042","https://openalex.org/W2183585111","https://openalex.org/W35925242","https://openalex.org/W2253304054","https://openalex.org/W4243211268","https://openalex.org/W2571659485","https://openalex.org/W4285175229","https://openalex.org/W2164098295"],"abstract_inverted_index":{"An":[0],"increasing":[1],"number":[2],"of":[3],"software":[4,43],"applications":[5],"incorporate":[6],"machine":[7],"learning":[8],"(ML)":[9],"solutions":[10],"for":[11],"cognitive":[12,68],"tasks":[13],"that":[14,32],"statistically":[15],"mimic":[16],"human":[17,24,50],"behaviors.":[18],"To":[19],"test":[20],"such":[21],"software,":[22,37],"tremendous":[23],"effort":[25],"is":[26,44,56,59,65],"needed":[27],"to":[28,35,39],"design":[29],"image/text/audio":[30],"inputs":[31,47],"are":[33],"relevant":[34],"the":[36,42,63,67,72,75],"and":[38],"judge":[40],"whether":[41,62],"processing":[45],"these":[46],"as":[48],"most":[49],"beings":[51],"do.":[52],"Even":[53],"when":[54],"misbehavior":[55],"exposed,":[57],"it":[58],"often":[60],"unclear":[61],"culprit":[64],"inside":[66],"ML":[69],"API":[70],"or":[71],"code":[73],"using":[74],"API.":[76]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":7},{"year":2024,"cited_by_count":2},{"year":2023,"cited_by_count":8},{"year":2022,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}