{"id":"https://openalex.org/W4399601922","doi":"https://doi.org/10.1145/3672451","title":"Keeper: Automated Testing and Fixing of Machine Learning Software","display_name":"Keeper: Automated Testing and Fixing of Machine Learning Software","publication_year":2024,"publication_date":"2024-06-13","ids":{"openalex":"https://openalex.org/W4399601922","doi":"https://doi.org/10.1145/3672451"},"language":"en","primary_location":{"id":"doi:10.1145/3672451","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3672451","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3672451","source":{"id":"https://openalex.org/S142627899","display_name":"ACM Transactions on Software Engineering and Methodology","issn_l":"1049-331X","issn":["1049-331X","1557-7392"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Software Engineering and Methodology","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"bronze","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3672451","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5016446062","display_name":"Chengcheng Wan","orcid":"https://orcid.org/0000-0001-9162-9688"},"institutions":[{"id":"https://openalex.org/I66867065","display_name":"East China Normal University","ror":"https://ror.org/02n96ep67","country_code":"CN","type":"education","lineage":["https://openalex.org/I66867065"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Chengcheng Wan","raw_affiliation_strings":["East China Normal University, Shanghai, China","East China Normal University, China"],"affiliations":[{"raw_affiliation_string":"East China Normal University, Shanghai, China","institution_ids":["https://openalex.org/I66867065"]},{"raw_affiliation_string":"East China Normal University, China","institution_ids":["https://openalex.org/I66867065"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5058741126","display_name":"Shicheng Liu","orcid":"https://orcid.org/0000-0003-2697-1916"},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Shicheng Liu","raw_affiliation_strings":["Stanford University, Stanford, CA, USA","Stanford University, United States"],"affiliations":[{"raw_affiliation_string":"Stanford University, Stanford, CA, USA","institution_ids":["https://openalex.org/I97018004"]},{"raw_affiliation_string":"Stanford University, United States","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5060048530","display_name":"Sophie Xie","orcid":"https://orcid.org/0009-0000-5484-5234"},"institutions":[{"id":"https://openalex.org/I95457486","display_name":"University of California, Berkeley","ror":"https://ror.org/01an7q238","country_code":"US","type":"education","lineage":["https://openalex.org/I95457486"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Sophie Xie","raw_affiliation_strings":["University of California, Berkeley, CA, USA","University of California, Berkeley, United States"],"affiliations":[{"raw_affiliation_string":"University of California, Berkeley, CA, USA","institution_ids":["https://openalex.org/I95457486"]},{"raw_affiliation_string":"University of California, Berkeley, United States","institution_ids":["https://openalex.org/I95457486"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100350539","display_name":"Yuhan Liu","orcid":"https://orcid.org/0009-0002-5957-5071"},"institutions":[{"id":"https://openalex.org/I40347166","display_name":"University of Chicago","ror":"https://ror.org/024mw5h28","country_code":"US","type":"education","lineage":["https://openalex.org/I40347166"]},{"id":"https://openalex.org/I39422238","display_name":"University of Illinois Chicago","ror":"https://ror.org/02mpq6x41","country_code":"US","type":"education","lineage":["https://openalex.org/I39422238"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yuhan Liu","raw_affiliation_strings":["University of Chicago, Chicago, IL, USA","University of Chicago, United States"],"affiliations":[{"raw_affiliation_string":"University of Chicago, Chicago, IL, USA","institution_ids":["https://openalex.org/I40347166","https://openalex.org/I39422238"]},{"raw_affiliation_string":"University of Chicago, United States","institution_ids":["https://openalex.org/I40347166"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5080833704","display_name":"Henry Hoffmann","orcid":"https://orcid.org/0000-0003-0816-8150"},"institutions":[{"id":"https://openalex.org/I39422238","display_name":"University of Illinois Chicago","ror":"https://ror.org/02mpq6x41","country_code":"US","type":"education","lineage":["https://openalex.org/I39422238"]},{"id":"https://openalex.org/I40347166","display_name":"University of Chicago","ror":"https://ror.org/024mw5h28","country_code":"US","type":"education","lineage":["https://openalex.org/I40347166"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Henry Hoffmann","raw_affiliation_strings":["University of Chicago, Chicago, IL, USA","University of Chicago, United States"],"affiliations":[{"raw_affiliation_string":"University of Chicago, Chicago, IL, USA","institution_ids":["https://openalex.org/I40347166","https://openalex.org/I39422238"]},{"raw_affiliation_string":"University of Chicago, United States","institution_ids":["https://openalex.org/I40347166"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5001371942","display_name":"Michael Maire","orcid":"https://orcid.org/0000-0002-9778-6673"},"institutions":[{"id":"https://openalex.org/I39422238","display_name":"University of Illinois Chicago","ror":"https://ror.org/02mpq6x41","country_code":"US","type":"education","lineage":["https://openalex.org/I39422238"]},{"id":"https://openalex.org/I40347166","display_name":"University of Chicago","ror":"https://ror.org/024mw5h28","country_code":"US","type":"education","lineage":["https://openalex.org/I40347166"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Michael Maire","raw_affiliation_strings":["University of Chicago, Chicago, IL, USA","University of Chicago, United States"],"affiliations":[{"raw_affiliation_string":"University of Chicago, Chicago, IL, USA","institution_ids":["https://openalex.org/I40347166","https://openalex.org/I39422238"]},{"raw_affiliation_string":"University of Chicago, United States","institution_ids":["https://openalex.org/I40347166"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5061448314","display_name":"Shan Lu","orcid":"https://orcid.org/0000-0002-0757-4600"},"institutions":[{"id":"https://openalex.org/I40347166","display_name":"University of Chicago","ror":"https://ror.org/024mw5h28","country_code":"US","type":"education","lineage":["https://openalex.org/I40347166"]},{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Shan Lu","raw_affiliation_strings":["Microsoft Research and University of Chicago, Chicago, IL, USA","Microsoft Research and University of Chicago, United States"],"affiliations":[{"raw_affiliation_string":"Microsoft Research and University of Chicago, Chicago, IL, USA","institution_ids":["https://openalex.org/I1290206253","https://openalex.org/I40347166"]},{"raw_affiliation_string":"Microsoft Research and University of Chicago, United States","institution_ids":["https://openalex.org/I40347166","https://openalex.org/I1290206253"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":7,"corresponding_author_ids":["https://openalex.org/A5016446062"],"corresponding_institution_ids":["https://openalex.org/I66867065"],"apc_list":null,"apc_paid":null,"fwci":0.7274,"has_fulltext":false,"cited_by_count":2,"citation_normalized_percentile":{"value":0.74321063,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":95,"max":96},"biblio":{"volume":"33","issue":"7","first_page":"1","last_page":"33"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11241","display_name":"Advanced Malware Detection Techniques","score":0.9973000288009644,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9968000054359436,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8500792384147644},{"id":"https://openalex.org/keywords/correctness","display_name":"Correctness","score":0.6292185187339783},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.4948362708091736},{"id":"https://openalex.org/keywords/software-testing","display_name":"Software testing","score":0.4858507812023163},{"id":"https://openalex.org/keywords/symbolic-execution","display_name":"Symbolic execution","score":0.47889652848243713},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.4530505836009979},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.34457504749298096},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.3260851204395294},{"id":"https://openalex.org/keywords/embedded-system","display_name":"Embedded system","score":0.32402899861335754}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8500792384147644},{"id":"https://openalex.org/C55439883","wikidata":"https://www.wikidata.org/wiki/Q360812","display_name":"Correctness","level":2,"score":0.6292185187339783},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.4948362708091736},{"id":"https://openalex.org/C2984328558","wikidata":"https://www.wikidata.org/wiki/Q188522","display_name":"Software testing","level":3,"score":0.4858507812023163},{"id":"https://openalex.org/C2779639559","wikidata":"https://www.wikidata.org/wiki/Q7661178","display_name":"Symbolic execution","level":3,"score":0.47889652848243713},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.4530505836009979},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.34457504749298096},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.3260851204395294},{"id":"https://openalex.org/C149635348","wikidata":"https://www.wikidata.org/wiki/Q193040","display_name":"Embedded system","level":1,"score":0.32402899861335754}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3672451","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3672451","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3672451","source":{"id":"https://openalex.org/S142627899","display_name":"ACM Transactions on Software Engineering and Methodology","issn_l":"1049-331X","issn":["1049-331X","1557-7392"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Software Engineering and Methodology","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1145/3672451","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3672451","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3672451","source":{"id":"https://openalex.org/S142627899","display_name":"ACM Transactions on Software Engineering and Methodology","issn_l":"1049-331X","issn":["1049-331X","1557-7392"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Software Engineering and Methodology","raw_type":"journal-article"},"sustainable_development_goals":[{"score":0.550000011920929,"id":"https://metadata.un.org/sdg/16","display_name":"Peace, Justice and strong institutions"}],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":false},"content_urls":{"pdf":"https://content.openalex.org/works/W4399601922.pdf"},"referenced_works_count":86,"referenced_works":["https://openalex.org/W2078945459","https://openalex.org/W2108598243","https://openalex.org/W2113730959","https://openalex.org/W2137406659","https://openalex.org/W2157309873","https://openalex.org/W2176208359","https://openalex.org/W2182687482","https://openalex.org/W2405754747","https://openalex.org/W2518851919","https://openalex.org/W2522956255","https://openalex.org/W2552408584","https://openalex.org/W2600531987","https://openalex.org/W2616028256","https://openalex.org/W2730550703","https://openalex.org/W2739781989","https://openalex.org/W2750930819","https://openalex.org/W2757656223","https://openalex.org/W2775678306","https://openalex.org/W2795274753","https://openalex.org/W2799640043","https://openalex.org/W2859484040","https://openalex.org/W2885205917","https://openalex.org/W2887009404","https://openalex.org/W2887649345","https://openalex.org/W2888307014","https://openalex.org/W2888483524","https://openalex.org/W2898851569","https://openalex.org/W2898868990","https://openalex.org/W2911282308","https://openalex.org/W2922234936","https://openalex.org/W2946581789","https://openalex.org/W2954903132","https://openalex.org/W2957905354","https://openalex.org/W2963327228","https://openalex.org/W2963417953","https://openalex.org/W2972359262","https://openalex.org/W2977524784","https://openalex.org/W2982540258","https://openalex.org/W2991533980","https://openalex.org/W2994987245","https://openalex.org/W2998795119","https://openalex.org/W2999072931","https://openalex.org/W2999193264","https://openalex.org/W2999450509","https://openalex.org/W2999691867","https://openalex.org/W3017068954","https://openalex.org/W3035671939","https://openalex.org/W3042650574","https://openalex.org/W3042703469","https://openalex.org/W3043199767","https://openalex.org/W3043201130","https://openalex.org/W3043238989","https://openalex.org/W3043716470","https://openalex.org/W3043746006","https://openalex.org/W3090423701","https://openalex.org/W3091388282","https://openalex.org/W3091407209","https://openalex.org/W3091633490","https://openalex.org/W3101094684","https://openalex.org/W3103158293","https://openalex.org/W3104896058","https://openalex.org/W3105249702","https://openalex.org/W3109004940","https://openalex.org/W3123045479","https://openalex.org/W3125205424","https://openalex.org/W3143148979","https://openalex.org/W3169232032","https://openalex.org/W3207613969","https://openalex.org/W4224249875","https://openalex.org/W4225320712","https://openalex.org/W4238623774","https://openalex.org/W4239084643","https://openalex.org/W4239789687","https://openalex.org/W4246691913","https://openalex.org/W4251934665","https://openalex.org/W4255086533","https://openalex.org/W4255751284","https://openalex.org/W4284669642","https://openalex.org/W4284687826","https://openalex.org/W4284702108","https://openalex.org/W4284707748","https://openalex.org/W4286530316","https://openalex.org/W4288083516","https://openalex.org/W4296964121","https://openalex.org/W4313563862","https://openalex.org/W4384345647"],"related_works":["https://openalex.org/W3168910894","https://openalex.org/W4239107164","https://openalex.org/W4385965400","https://openalex.org/W4388483831","https://openalex.org/W2090301720","https://openalex.org/W2008194781","https://openalex.org/W3089971865","https://openalex.org/W4319996264","https://openalex.org/W2038209453","https://openalex.org/W3176274227"],"abstract_inverted_index":{"The":[0,88],"increasing":[1],"number":[2],"of":[3,91,113,159,181,190,194],"software":[4,22,37,135],"applications":[5,160],"incorporating":[6],"machine":[7],"learning":[8],"(ML)":[9],"solutions":[10],"has":[11],"led":[12],"to":[13,27,35,41,67,125,143,150],"the":[14,55,62,69,100,152],"need":[15],"for":[16,85],"testing":[17,20,81],"techniques.":[18],"However,":[19],"ML":[21,59,86,102,146],"requires":[23],"tremendous":[24],"human":[25,42,111],"effort":[26],"design":[28],"realistic":[29],"and":[30,34,65,82,108,174,192,200],"relevant":[31],"test":[32],"inputs":[33],"judge":[36],"output":[38],"correctness":[39],"according":[40],"common":[43,110],"sense.":[44],"Even":[45],"when":[46],"misbehavior":[47,138],"is":[48,51,57,93,139],"exposed,":[49,140],"it":[50],"often":[52],"unclear":[53],"whether":[54],"defect":[56],"inside":[58],"API":[60],"or":[61],"surrounding":[63],"code":[64,131,176],"how":[66,145],"fix":[68],"implementation.":[70],"This":[71],"article":[72],"tackles":[73],"these":[74,118],"challenges":[75],"by":[76],"proposing":[77],"Keeper,":[78],"an":[79,105],"automated":[80],"fixing":[83,201],"tool":[84],"software.":[87],"core":[89],"idea":[90],"Keeper":[92,128,141,163],"designing":[94],"pseudo-inverse":[95],"functions":[96,119],"that":[97,133,162,188],"semantically":[98],"reverse":[99],"corresponding":[101],"task":[103],"in":[104],"empirical":[106],"way":[107],"proxy":[109],"judgment":[112],"real-world":[114],"data.":[115],"It":[116],"incorporates":[117],"into":[120],"a":[121,157],"symbolic":[122],"execution":[123],"engine":[124],"generate":[126],"tests.":[127],"also":[129],"detects":[130],"smells":[132,177],"degrade":[134],"performance.":[136],"Once":[137],"attempts":[142],"change":[144],"APIs":[147],"are":[148],"used":[149],"alleviate":[151],"misbehavior.":[153],"Our":[154,184],"evaluation":[155],"on":[156],"variety":[158],"shows":[161],"greatly":[164],"improves":[165],"branch":[166],"coverage,":[167],"while":[168],"identifying":[169],"74":[170],"previously":[171],"unknown":[172],"failures":[173],"19":[175],"from":[178],"56":[179],"out":[180],"104":[182],"applications.":[183],"user":[185],"studies":[186],"show":[187],"78%":[189],"end-users":[191],"95%":[193],"developers":[195],"agree":[196],"with":[197],"Keeper\u2019s":[198],"detection":[199],"results.":[202]},"counts_by_year":[{"year":2025,"cited_by_count":2}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}