{"id":"https://openalex.org/W3095962339","doi":"https://doi.org/10.1109/icse-seip52600.2021.00036","title":"What It Would Take to Use Mutation Testing in Industry\u2014A Study at Facebook","display_name":"What It Would Take to Use Mutation Testing in Industry\u2014A Study at Facebook","publication_year":2021,"publication_date":"2021-05-01","ids":{"openalex":"https://openalex.org/W3095962339","doi":"https://doi.org/10.1109/icse-seip52600.2021.00036","mag":"3095962339"},"language":"en","primary_location":{"id":"doi:10.1109/icse-seip52600.2021.00036","doi":"https://doi.org/10.1109/icse-seip52600.2021.00036","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icse-seip52600.2021.00036","pdf_url":null,"source":{"id":"https://openalex.org/S7407089274","display_name":"IEEE/ACM International Conference on Software Engineering: Software Engineering in Practice (ICSE-SEIP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2021 IEEE/ACM 43rd International Conference on Software Engineering: Software Engineering in Practice (ICSE-SEIP)","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","is_traditional":true,"indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2010.13464","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5021746976","display_name":"Moritz Beller","orcid":"https://orcid.org/0000-0003-4852-0526","observed_orcids":["https://orcid.org/0000-0003-4852-0526"]},"institutions":[{"id":"https://openalex.org/I4210114444","display_name":"Meta (United States)","ror":"https://ror.org/01zbnvs85","country_code":"US","type":"company","lineage":["https://openalex.org/I4210114444"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Moritz Beller","raw_affiliation_strings":["Probability, Facebook, Inc, Menlo Park, USA","Facebook, Inc., USA#TAB#"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Probability, Facebook, Inc, Menlo Park, USA","institution_ids":["https://openalex.org/I4210114444"]},{"raw_affiliation_string":"Facebook, Inc., USA#TAB#","institution_ids":["https://openalex.org/I4210114444"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5000845080","display_name":"Chu-Pan Wong","orcid":"https://orcid.org/0000-0003-3694-9077","observed_orcids":["https://orcid.org/0000-0003-3694-9077"]},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Chu-Pan Wong","raw_affiliation_strings":["Carnegie Mellon University (work done while interning at Facebook), Pittsburgh, USA","Carnegie -Mellon University, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University (work done while interning at Facebook), Pittsburgh, USA","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"Carnegie -Mellon University, USA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Johannes Bader","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4401726835","display_name":"Jane Street (United States)","ror":"https://ror.org/05p5v2j38","country_code":"US","type":"company","lineage":["https://openalex.org/I4401726835"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Johannes Bader","raw_affiliation_strings":["Jane Street Capital (ex-employee; work done while at Facebook)","Jane Street Capital"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Jane Street Capital (ex-employee; work done while at Facebook)","institution_ids":["https://openalex.org/I4401726835"]},{"raw_affiliation_string":"Jane Street Capital","institution_ids":["https://openalex.org/I4401726835"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5013415938","display_name":"Andrew Scott","orcid":"https://orcid.org/0000-0002-6784-5652","observed_orcids":["https://orcid.org/0000-0002-6784-5652"]},"institutions":[{"id":"https://openalex.org/I4210114444","display_name":"Meta (United States)","ror":"https://ror.org/01zbnvs85","country_code":"US","type":"company","lineage":["https://openalex.org/I4210114444"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Andrew Scott","raw_affiliation_strings":["Probability, Facebook, Inc, Menlo Park, USA","Facebook, Inc., USA#TAB#"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Probability, Facebook, Inc, Menlo Park, USA","institution_ids":["https://openalex.org/I4210114444"]},{"raw_affiliation_string":"Facebook, Inc., USA#TAB#","institution_ids":["https://openalex.org/I4210114444"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5001038577","display_name":"Mateusz Machalica","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210114444","display_name":"Meta (United States)","ror":"https://ror.org/01zbnvs85","country_code":"US","type":"company","lineage":["https://openalex.org/I4210114444"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Mateusz Machalica","raw_affiliation_strings":["Probability, Facebook, Inc, Menlo Park, USA","Facebook, Inc., USA#TAB#"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Probability, Facebook, Inc, Menlo Park, USA","institution_ids":["https://openalex.org/I4210114444"]},{"raw_affiliation_string":"Facebook, Inc., USA#TAB#","institution_ids":["https://openalex.org/I4210114444"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101965118","display_name":"Satish Chandra","orcid":"https://orcid.org/0000-0003-2546-9000","observed_orcids":["https://orcid.org/0000-0003-2546-9000"]},"institutions":[{"id":"https://openalex.org/I4210114444","display_name":"Meta (United States)","ror":"https://ror.org/01zbnvs85","country_code":"US","type":"company","lineage":["https://openalex.org/I4210114444"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Satish Chandra","raw_affiliation_strings":["Probability, Facebook, Inc, Menlo Park, USA","Facebook, Inc., USA#TAB#"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Probability, Facebook, Inc, Menlo Park, USA","institution_ids":["https://openalex.org/I4210114444"]},{"raw_affiliation_string":"Facebook, Inc., USA#TAB#","institution_ids":["https://openalex.org/I4210114444"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5053775941","display_name":"Erik Meijer","orcid":"https://orcid.org/0000-0001-7959-4670","observed_orcids":["https://orcid.org/0000-0001-7959-4670","https://orcid.org/0000-0001-6779-0295","https://orcid.org/0000-0002-4238-5098"]},"institutions":[{"id":"https://openalex.org/I4210114444","display_name":"Meta (United States)","ror":"https://ror.org/01zbnvs85","country_code":"US","type":"company","lineage":["https://openalex.org/I4210114444"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Erik Meijer","raw_affiliation_strings":["Probability, Facebook, Inc, Menlo Park, USA","Facebook, Inc., USA#TAB#"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Probability, Facebook, Inc, Menlo Park, USA","institution_ids":["https://openalex.org/I4210114444"]},{"raw_affiliation_string":"Facebook, Inc., USA#TAB#","institution_ids":["https://openalex.org/I4210114444"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":8.321,"has_fulltext":true,"cited_by_count":73,"citation_normalized_percentile":{"value":0.98844673,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":92,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"268","last_page":"277"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":9.999999747378752e-05,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12127","display_name":"Software System Performance and Reliability","score":0.0,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/mutation-testing","display_name":"mutation testing","score":0.9980000257492065},{"id":"https://openalex.org/keywords/instrumentation","display_name":"instrumentation","score":0.4449999928474426},{"id":"https://openalex.org/keywords/test-suite-coverage","display_name":"test suite coverage","score":0.6899999976158142},{"id":"https://openalex.org/keywords/developer-behavior","display_name":"developer behavior","score":0.4129999876022339},{"id":"https://openalex.org/keywords/facebook","display_name":"Facebook","score":0.36500000953674316},{"id":"https://openalex.org/keywords/industrial-software-testing","display_name":"industrial software testing","score":0.7009999752044678},{"id":"https://openalex.org/keywords/test-adequacy","display_name":"test adequacy","score":0.4020000100135803}],"concepts":[{"id":"https://openalex.org/C151552104","wikidata":"https://www.wikidata.org/wiki/Q7705809","display_name":"Test suite","level":4,"score":0.6729925274848938},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6686993837356567},{"id":"https://openalex.org/C179518139","wikidata":"https://www.wikidata.org/wiki/Q5140297","display_name":"Coding (social sciences)","level":2,"score":0.5858184099197388},{"id":"https://openalex.org/C148027188","wikidata":"https://www.wikidata.org/wiki/Q907375","display_name":"Unit testing","level":3,"score":0.5659777522087097},{"id":"https://openalex.org/C79581498","wikidata":"https://www.wikidata.org/wiki/Q1367530","display_name":"Suite","level":2,"score":0.5613108277320862},{"id":"https://openalex.org/C548217200","wikidata":"https://www.wikidata.org/wiki/Q251","display_name":"Java","level":2,"score":0.5206838846206665},{"id":"https://openalex.org/C501734568","wikidata":"https://www.wikidata.org/wiki/Q42918","display_name":"Mutation","level":3,"score":0.49339810013771057},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.4742105305194855},{"id":"https://openalex.org/C143065580","wikidata":"https://www.wikidata.org/wiki/Q3285695","display_name":"Mutant","level":3,"score":0.4721037447452545},{"id":"https://openalex.org/C136197465","wikidata":"https://www.wikidata.org/wiki/Q1729295","display_name":"Variety (cybernetics)","level":2,"score":0.4664086103439331},{"id":"https://openalex.org/C163565370","wikidata":"https://www.wikidata.org/wiki/Q4308623","display_name":"Mutation testing","level":4,"score":0.44015079736709595},{"id":"https://openalex.org/C128942645","wikidata":"https://www.wikidata.org/wiki/Q1568346","display_name":"Test case","level":3,"score":0.35286179184913635},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.32397717237472534},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2874397337436676},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.26961246132850647},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.22257933020591736},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.17391836643218994},{"id":"https://openalex.org/C54355233","wikidata":"https://www.wikidata.org/wiki/Q7162","display_name":"Genetics","level":1,"score":0.13600891828536987},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.13308408856391907},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.10781785845756531},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.09177237749099731},{"id":"https://openalex.org/C104317684","wikidata":"https://www.wikidata.org/wiki/Q7187","display_name":"Gene","level":2,"score":0.0},{"id":"https://openalex.org/C152877465","wikidata":"https://www.wikidata.org/wiki/Q208042","display_name":"Regression analysis","level":2,"score":0.0},{"id":"https://openalex.org/C166957645","wikidata":"https://www.wikidata.org/wiki/Q23498","display_name":"Archaeology","level":1,"score":0.0},{"id":"https://openalex.org/C95457728","wikidata":"https://www.wikidata.org/wiki/Q309","display_name":"History","level":0,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1109/icse-seip52600.2021.00036","doi":"https://doi.org/10.1109/icse-seip52600.2021.00036","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icse-seip52600.2021.00036","pdf_url":null,"source":{"id":"https://openalex.org/S7407089274","display_name":"IEEE/ACM International Conference on Software Engineering: Software Engineering in Practice (ICSE-SEIP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2021 IEEE/ACM 43rd International Conference on Software Engineering: Software Engineering in Practice (ICSE-SEIP)","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2010.13464","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2010.13464","pdf_url":"https://arxiv.org/pdf/2010.13464","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2010.13464","doi":"https://doi.org/10.48550/arxiv.2010.13464","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2010.13464","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2010.13464","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2010.13464","pdf_url":"https://arxiv.org/pdf/2010.13464","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3095962339.pdf","grobid_xml":"https://content.openalex.org/works/W3095962339.grobid-xml"},"referenced_works_count":31,"referenced_works":["https://openalex.org/W814172419","https://openalex.org/W1560412976","https://openalex.org/W1567503821","https://openalex.org/W1996701120","https://openalex.org/W2002072718","https://openalex.org/W2018017297","https://openalex.org/W2027565420","https://openalex.org/W2049695835","https://openalex.org/W2050264635","https://openalex.org/W2051400406","https://openalex.org/W2063068372","https://openalex.org/W2075592454","https://openalex.org/W2076609667","https://openalex.org/W2080097084","https://openalex.org/W2106072155","https://openalex.org/W2108557864","https://openalex.org/W2135841285","https://openalex.org/W2135937266","https://openalex.org/W2146957318","https://openalex.org/W2156723666","https://openalex.org/W2251980239","https://openalex.org/W2725449579","https://openalex.org/W2734519371","https://openalex.org/W2739564891","https://openalex.org/W2786944871","https://openalex.org/W2794756405","https://openalex.org/W2888728013","https://openalex.org/W2897352130","https://openalex.org/W2914161001","https://openalex.org/W3143303760","https://openalex.org/W4249448758"],"related_works":["https://openalex.org/W3196350964","https://openalex.org/W6887968514","https://openalex.org/W3129295186","https://openalex.org/W4232307874","https://openalex.org/W2763110557","https://openalex.org/W2884023932","https://openalex.org/W3136164051","https://openalex.org/W1677845387","https://openalex.org/W4403520192","https://openalex.org/W2950639681"],"abstract_inverted_index":{"Traditionally,":[0],"mutation":[1,136,183],"testing":[2,141],"generates":[3],"an":[4,162],"abundance":[5],"of":[6,9,21,63,93,107,116,140,148,176,182],"small":[7],"deviations":[8],"a":[10,61,124,138,166,174,187],"program,":[11],"called":[12],"mutants.":[13,204],"At":[14],"industrial":[15],"systems":[16],"the":[17,33,80,90,108,135,149,155,180,199],"scale":[18],"and":[19,68,119],"size":[20],"Facebook's,":[22],"doing":[23],"this":[24,50,53],"is":[25],"infeasible.":[26],"We":[27,78],"should":[28],"not":[29,172],"create":[30],"mutants":[31,101,110],"that":[32,41,71,84,103,134],"test":[34,114],"suite":[35,115],"would":[36,151],"likely":[37],"fail":[38],"on":[39,96,154,203],"or":[40,164],"give":[42],"no":[43],"actionable":[44],"signal":[45],"to":[46,158,197],"developers.":[47],"To":[48],"tackle":[49],"problem,":[51],"in":[52,123,142],"paper,":[54],"we":[55,191],"semi-automatically":[56],"learn":[57],"error-inducing":[58],"patterns":[59],"from":[60,69],"corpus":[62],"common":[64],"Java":[65],"coding":[66],"errors":[67],"changes":[70],"caused":[72],"operational":[73],"anomalies":[74],"at":[75],"Facebook":[76],"specifically.":[77],"combine":[79],"mutations":[81],"with":[82,127],"instrumentation":[83],"measures":[85],"which":[86],"tests":[87],"exactly":[88],"visited":[89],"mutated":[91],"piece":[92],"code.":[94],"Results":[95],"more":[97,104],"than":[98,105],"15,000":[99],"generated":[100,109],"show":[102],"half":[106,147],"survive":[111],"Facebook's":[112],"rigorous":[113],"unit,":[117],"integration,":[118],"system":[120],"tests.":[121],"Moreover,":[122],"case":[125],"study":[126],"26":[128,150],"developers,":[129],"all":[130],"but":[131],"two":[132],"expressed":[133],"exposed":[137],"lack":[139],"principle.":[143],"As":[144],"such,":[145],"almost":[146],"actually":[152],"act":[153],"mutant":[156],"presented":[157],"them":[159],"by":[160],"adapting":[161],"existing":[163],"creating":[165],"new":[167],"test.":[168],"The":[169],"others":[170],"did":[171],"for":[173],"variety":[175],"reasons":[177],"often":[178],"outside":[179],"scope":[181],"testing.":[184],"It":[185],"remains":[186],"practical":[188],"challenge":[189],"how":[190],"can":[192],"include":[193],"such":[194],"external":[195],"information":[196],"increase":[198],"true":[200],"actionability":[201],"rate":[202]},"counts_by_year":[{"year":2026,"cited_by_count":12},{"year":2025,"cited_by_count":14},{"year":2024,"cited_by_count":15},{"year":2023,"cited_by_count":18},{"year":2022,"cited_by_count":12},{"year":2021,"cited_by_count":2}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2020-11-09T00:00:00"}