{"id":"https://openalex.org/W3044741787","doi":"https://doi.org/10.1145/3394486.3403379","title":"An Empirical Analysis of Backward Compatibility in Machine Learning Systems","display_name":"An Empirical Analysis of Backward Compatibility in Machine Learning Systems","publication_year":2020,"publication_date":"2020-08-20","ids":{"openalex":"https://openalex.org/W3044741787","doi":"https://doi.org/10.1145/3394486.3403379","mag":"3044741787"},"language":"en","primary_location":{"id":"doi:10.1145/3394486.3403379","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3394486.3403379","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 26th ACM SIGKDD International Conference on Knowledge Discovery & Data Mining","raw_type":"proceedings-article"},"type":"preprint","indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2008.04572","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5043099018","display_name":"Megha Srivastava","orcid":"https://orcid.org/0009-0002-7816-8391"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Megha Srivastava","raw_affiliation_strings":["Microsoft Research, Redmond, WA, USA","Microsoft Research, redmond, WA, USA#TAB#"],"affiliations":[{"raw_affiliation_string":"Microsoft Research, Redmond, WA, USA","institution_ids":["https://openalex.org/I1290206253"]},{"raw_affiliation_string":"Microsoft Research, redmond, WA, USA#TAB#","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5011998621","display_name":"Besmira Nushi","orcid":"https://orcid.org/0000-0002-7554-8586"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]},{"id":"https://openalex.org/I1321014770","display_name":"Association for Computing Machinery","ror":"https://ror.org/03wsadn68","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I1321014770"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Besmira Nushi","raw_affiliation_strings":["Microsoft Research, Redmond, WA, USA","Association for Computing Machinery"],"affiliations":[{"raw_affiliation_string":"Microsoft Research, Redmond, WA, USA","institution_ids":["https://openalex.org/I1290206253"]},{"raw_affiliation_string":"Association for Computing Machinery","institution_ids":["https://openalex.org/I1321014770"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5028114802","display_name":"Ece Kamar","orcid":null},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ece Kamar","raw_affiliation_strings":["Microsoft Research, Redmond, WA, USA","Microsoft Research, redmond, WA, USA#TAB#"],"affiliations":[{"raw_affiliation_string":"Microsoft Research, Redmond, WA, USA","institution_ids":["https://openalex.org/I1290206253"]},{"raw_affiliation_string":"Microsoft Research, redmond, WA, USA#TAB#","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101671828","display_name":"Shital Shah","orcid":"https://orcid.org/0000-0002-8995-1704"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Shital Shah","raw_affiliation_strings":["Microsoft Research, Redmond, WA, USA","Microsoft Research, redmond, WA, USA#TAB#"],"affiliations":[{"raw_affiliation_string":"Microsoft Research, Redmond, WA, USA","institution_ids":["https://openalex.org/I1290206253"]},{"raw_affiliation_string":"Microsoft Research, redmond, WA, USA#TAB#","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5043228682","display_name":"Eric Horvitz","orcid":"https://orcid.org/0000-0002-8823-0614"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Eric Horvitz","raw_affiliation_strings":["Microsoft Research, Redmond, WA, USA","Microsoft Research, redmond, WA, USA#TAB#"],"affiliations":[{"raw_affiliation_string":"Microsoft Research, Redmond, WA, USA","institution_ids":["https://openalex.org/I1290206253"]},{"raw_affiliation_string":"Microsoft Research, redmond, WA, USA#TAB#","institution_ids":["https://openalex.org/I1290206253"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5043099018"],"corresponding_institution_ids":["https://openalex.org/I1290206253"],"apc_list":null,"apc_paid":null,"fwci":0.136,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":{"value":0.54936522,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":94},"biblio":{"volume":null,"issue":null,"first_page":"3272","last_page":"3280"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9970999956130981,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.9968000054359436,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/compatibility","display_name":"Compatibility (geochemistry)","score":0.8029996156692505},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7524288892745972},{"id":"https://openalex.org/keywords/robustness","display_name":"Robustness (evolution)","score":0.524667501449585},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.5000357627868652},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4446667432785034},{"id":"https://openalex.org/keywords/cloud-computing","display_name":"Cloud computing","score":0.4408394694328308},{"id":"https://openalex.org/keywords/training-set","display_name":"Training set","score":0.4376249313354492},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.3464123606681824},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.3400878310203552},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.10446429252624512}],"concepts":[{"id":"https://openalex.org/C2778648169","wikidata":"https://www.wikidata.org/wiki/Q967768","display_name":"Compatibility (geochemistry)","level":2,"score":0.8029996156692505},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7524288892745972},{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.524667501449585},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5000357627868652},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4446667432785034},{"id":"https://openalex.org/C79974875","wikidata":"https://www.wikidata.org/wiki/Q483639","display_name":"Cloud computing","level":2,"score":0.4408394694328308},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.4376249313354492},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3464123606681824},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.3400878310203552},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.10446429252624512},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C104317684","wikidata":"https://www.wikidata.org/wiki/Q7187","display_name":"Gene","level":2,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C55493867","wikidata":"https://www.wikidata.org/wiki/Q7094","display_name":"Biochemistry","level":1,"score":0.0},{"id":"https://openalex.org/C42360764","wikidata":"https://www.wikidata.org/wiki/Q83588","display_name":"Chemical engineering","level":1,"score":0.0}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.1145/3394486.3403379","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3394486.3403379","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 26th ACM SIGKDD International Conference on Knowledge Discovery & Data Mining","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2008.04572","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2008.04572","pdf_url":"https://arxiv.org/pdf/2008.04572","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"mag:3044741787","is_oa":true,"landing_page_url":"http://export.arxiv.org/pdf/2008.04572","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null},{"id":"doi:10.48550/arxiv.2008.04572","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2008.04572","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2008.04572","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2008.04572","pdf_url":"https://arxiv.org/pdf/2008.04572","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/17","score":0.4300000071525574,"display_name":"Partnerships for the goals"}],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3044741787.pdf","grobid_xml":"https://content.openalex.org/works/W3044741787.grobid-xml"},"referenced_works_count":39,"referenced_works":["https://openalex.org/W979089999","https://openalex.org/W1579279110","https://openalex.org/W1603565383","https://openalex.org/W1610496399","https://openalex.org/W1682403713","https://openalex.org/W1866072925","https://openalex.org/W2046298800","https://openalex.org/W2060277733","https://openalex.org/W2098874108","https://openalex.org/W2110986222","https://openalex.org/W2113290770","https://openalex.org/W2113459411","https://openalex.org/W2113839990","https://openalex.org/W2149754870","https://openalex.org/W2167460663","https://openalex.org/W2189162242","https://openalex.org/W2287418003","https://openalex.org/W2407420912","https://openalex.org/W2580550478","https://openalex.org/W2591700809","https://openalex.org/W2592335154","https://openalex.org/W2743151379","https://openalex.org/W2767164197","https://openalex.org/W2810918175","https://openalex.org/W2884061367","https://openalex.org/W2885593519","https://openalex.org/W2886614482","https://openalex.org/W2903996579","https://openalex.org/W2905034244","https://openalex.org/W2922234936","https://openalex.org/W2949157943","https://openalex.org/W2952602496","https://openalex.org/W2953027140","https://openalex.org/W2963060032","https://openalex.org/W2964253222","https://openalex.org/W2964317695","https://openalex.org/W3005940936","https://openalex.org/W3118608800","https://openalex.org/W6636177537"],"related_works":["https://openalex.org/W3081036064","https://openalex.org/W2771505108","https://openalex.org/W2610436371","https://openalex.org/W2040264572","https://openalex.org/W3204385750","https://openalex.org/W2787456385","https://openalex.org/W2301778277","https://openalex.org/W2003775793","https://openalex.org/W2337409001","https://openalex.org/W2333794827","https://openalex.org/W289025794","https://openalex.org/W2055058987","https://openalex.org/W2792558794","https://openalex.org/W2763776913","https://openalex.org/W3166496577","https://openalex.org/W2016637377","https://openalex.org/W2097541254","https://openalex.org/W1525301981","https://openalex.org/W1530638770","https://openalex.org/W1580478024"],"abstract_inverted_index":{"In":[0],"many":[1],"applications":[2],"of":[3,13,93,166],"machine":[4],"learning":[5],"(ML),":[6],"updates":[7,62],"are":[8],"performed":[9],"with":[10,103,119,170],"the":[11,85,91,174],"goal":[12],"enhancing":[14],"model":[15,160],"performance.":[16],"However,":[17],"current":[18],"practices":[19],"for":[20,96,176],"updating":[21],"models":[22,64],"rely":[23],"solely":[24],"on":[25,113,146],"isolated,":[26],"aggregate":[27],"performance":[28],"analyses,":[29],"overlooking":[30],"important":[31],"dependencies,":[32],"expectations,":[33],"and":[34,58,110,122,163,180],"needs":[35],"in":[36,63,66,79,125,152,155],"real-world":[37],"deployments.":[38],"We":[39,100],"consider":[40],"how":[41],"updates,":[42],"intended":[43],"to":[44,84,141],"improve":[45],"ML":[46,108,123],"models,":[47],"can":[48,53,74],"introduce":[49],"new":[50],"errors":[51],"that":[52,81,131],"significantly":[54],"affect":[55],"downstream":[56],"systems":[57,80],"users.":[59],"For":[60],"example,":[61],"used":[65],"cloud-based":[67],"classification":[68],"services,":[69],"such":[70],"as":[71],"image":[72],"recognition,":[73],"cause":[75],"unexpected":[76],"erroneous":[77],"behavior":[78],"make":[82],"calls":[83],"services.":[86],"Prior":[87],"work":[88],"has":[89],"shown":[90],"importance":[92],"\"backward":[94],"compatibility\"":[95],"maintaining":[97],"human":[98],"trust.":[99],"study":[101],"challenges":[102],"backward":[104,156],"compatibility":[105,133,157,177],"across":[106],"different":[107],"architectures":[109],"datasets,":[111],"focusing":[112],"common":[114],"settings":[115],"including":[116],"data":[117,138],"shifts":[118],"structured":[120],"noise":[121,171],"employed":[124],"inferential":[126],"pipelines.":[127],"Our":[128],"results":[129,151],"show":[130],"(i)":[132],"issues":[134],"arise":[135],"even":[136,158],"without":[137],"shift":[139],"due":[140],"optimization":[142],"stochasticity,":[143],"(ii)":[144],"training":[145],"large-scale":[147],"noisy":[148],"datasets":[149],"often":[150],"significant":[153],"decreases":[154],"when":[159],"accuracy":[161],"increases,":[162],"(iii)":[164],"distributions":[165],"incompatible":[167],"points":[168],"align":[169],"bias,":[172],"motivating":[173],"need":[175],"aware":[178],"de-noising":[179],"robustness":[181],"methods.":[182]},"counts_by_year":[{"year":2023,"cited_by_count":1}],"updated_date":"2026-03-11T14:59:36.786465","created_date":"2025-10-10T00:00:00"}