{"id":"https://openalex.org/W3196350964","doi":"https://doi.org/10.1109/tse.2021.3107634","title":"Practical Mutation Testing at Scale: A view from Google","display_name":"Practical Mutation Testing at Scale: A view from Google","publication_year":2021,"publication_date":"2021-08-27","ids":{"openalex":"https://openalex.org/W3196350964","doi":"https://doi.org/10.1109/tse.2021.3107634","mag":"3196350964"},"language":"en","primary_location":{"id":"doi:10.1109/tse.2021.3107634","is_oa":true,"landing_page_url":"https://doi.org/10.1109/tse.2021.3107634","pdf_url":"https://ieeexplore.ieee.org/ielx7/32/9923560/09524503.pdf","source":{"id":"https://openalex.org/S8351582","display_name":"IEEE Transactions on Software Engineering","issn_l":"0098-5589","issn":["0098-5589","1939-3520","2326-3881"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Software Engineering","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://ieeexplore.ieee.org/ielx7/32/9923560/09524503.pdf","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5102805229","display_name":"Goran Petrovi\u0107","orcid":"https://orcid.org/0000-0002-8056-7431"},"institutions":[{"id":"https://openalex.org/I4210100430","display_name":"Google (Switzerland)","ror":"https://ror.org/014f9c269","country_code":"CH","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210100430","https://openalex.org/I4210128969"]}],"countries":["CH"],"is_corresponding":true,"raw_author_name":"Goran Petrovic","raw_affiliation_strings":["Google LLC, Z&#x00FC;rich, Switzerland"],"affiliations":[{"raw_affiliation_string":"Google LLC, Z&#x00FC;rich, Switzerland","institution_ids":["https://openalex.org/I4210100430"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5075786120","display_name":"Marko Ivankovi\u0107","orcid":"https://orcid.org/0000-0002-8548-6008"},"institutions":[{"id":"https://openalex.org/I4210100430","display_name":"Google (Switzerland)","ror":"https://ror.org/014f9c269","country_code":"CH","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210100430","https://openalex.org/I4210128969"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Marko Ivankovic","raw_affiliation_strings":["Google LLC, Z&#x00FC;rich, Switzerland"],"affiliations":[{"raw_affiliation_string":"Google LLC, Z&#x00FC;rich, Switzerland","institution_ids":["https://openalex.org/I4210100430"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5079261847","display_name":"Gordon Fraser","orcid":"https://orcid.org/0000-0002-4364-6595"},"institutions":[{"id":"https://openalex.org/I186354981","display_name":"University of Passau","ror":"https://ror.org/05ydjnb78","country_code":"DE","type":"education","lineage":["https://openalex.org/I186354981"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Gordon Fraser","raw_affiliation_strings":["University of Passau, Passau, Germany"],"affiliations":[{"raw_affiliation_string":"University of Passau, Passau, Germany","institution_ids":["https://openalex.org/I186354981"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5088079823","display_name":"Ren\u00e9 Just","orcid":"https://orcid.org/0000-0002-5982-275X"},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Rene Just","raw_affiliation_strings":["University of Washington, Seattle, WA, USA"],"affiliations":[{"raw_affiliation_string":"University of Washington, Seattle, WA, USA","institution_ids":["https://openalex.org/I201448701"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5102805229"],"corresponding_institution_ids":["https://openalex.org/I4210100430"],"apc_list":null,"apc_paid":null,"fwci":13.6069,"has_fulltext":true,"cited_by_count":61,"citation_normalized_percentile":{"value":0.99319372,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":99,"max":100},"biblio":{"volume":"48","issue":"10","first_page":"3900","last_page":"3912"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9975000023841858,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11241","display_name":"Advanced Malware Detection Techniques","score":0.9930999875068665,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/mutation-testing","display_name":"Mutation testing","score":0.8180387020111084},{"id":"https://openalex.org/keywords/mutation","display_name":"Mutation","score":0.7509765625},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7452582120895386},{"id":"https://openalex.org/keywords/test-suite","display_name":"Test suite","score":0.7403067946434021},{"id":"https://openalex.org/keywords/codebase","display_name":"Codebase","score":0.7108731865882874},{"id":"https://openalex.org/keywords/scalability","display_name":"Scalability","score":0.5663204789161682},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.5183550119400024},{"id":"https://openalex.org/keywords/source-lines-of-code","display_name":"Source lines of code","score":0.4374357759952545},{"id":"https://openalex.org/keywords/source-code","display_name":"Source code","score":0.4246745705604553},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.41516929864883423},{"id":"https://openalex.org/keywords/test-case","display_name":"Test case","score":0.3385416865348816},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.32841333746910095},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.2756555676460266},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.2447042465209961},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.20137912034988403},{"id":"https://openalex.org/keywords/genetics","display_name":"Genetics","score":0.17516198754310608},{"id":"https://openalex.org/keywords/database","display_name":"Database","score":0.1571301519870758},{"id":"https://openalex.org/keywords/biology","display_name":"Biology","score":0.10746631026268005}],"concepts":[{"id":"https://openalex.org/C163565370","wikidata":"https://www.wikidata.org/wiki/Q4308623","display_name":"Mutation testing","level":4,"score":0.8180387020111084},{"id":"https://openalex.org/C501734568","wikidata":"https://www.wikidata.org/wiki/Q42918","display_name":"Mutation","level":3,"score":0.7509765625},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7452582120895386},{"id":"https://openalex.org/C151552104","wikidata":"https://www.wikidata.org/wiki/Q7705809","display_name":"Test suite","level":4,"score":0.7403067946434021},{"id":"https://openalex.org/C51929080","wikidata":"https://www.wikidata.org/wiki/Q2425187","display_name":"Codebase","level":3,"score":0.7108731865882874},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.5663204789161682},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.5183550119400024},{"id":"https://openalex.org/C199519371","wikidata":"https://www.wikidata.org/wiki/Q942695","display_name":"Source lines of code","level":3,"score":0.4374357759952545},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.4246745705604553},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.41516929864883423},{"id":"https://openalex.org/C128942645","wikidata":"https://www.wikidata.org/wiki/Q1568346","display_name":"Test case","level":3,"score":0.3385416865348816},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.32841333746910095},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.2756555676460266},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.2447042465209961},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.20137912034988403},{"id":"https://openalex.org/C54355233","wikidata":"https://www.wikidata.org/wiki/Q7162","display_name":"Genetics","level":1,"score":0.17516198754310608},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.1571301519870758},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.10746631026268005},{"id":"https://openalex.org/C104317684","wikidata":"https://www.wikidata.org/wiki/Q7187","display_name":"Gene","level":2,"score":0.0},{"id":"https://openalex.org/C152877465","wikidata":"https://www.wikidata.org/wiki/Q208042","display_name":"Regression analysis","level":2,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tse.2021.3107634","is_oa":true,"landing_page_url":"https://doi.org/10.1109/tse.2021.3107634","pdf_url":"https://ieeexplore.ieee.org/ielx7/32/9923560/09524503.pdf","source":{"id":"https://openalex.org/S8351582","display_name":"IEEE Transactions on Software Engineering","issn_l":"0098-5589","issn":["0098-5589","1939-3520","2326-3881"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Software Engineering","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1109/tse.2021.3107634","is_oa":true,"landing_page_url":"https://doi.org/10.1109/tse.2021.3107634","pdf_url":"https://ieeexplore.ieee.org/ielx7/32/9923560/09524503.pdf","source":{"id":"https://openalex.org/S8351582","display_name":"IEEE Transactions on Software Engineering","issn_l":"0098-5589","issn":["0098-5589","1939-3520","2326-3881"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Software Engineering","raw_type":"journal-article"},"sustainable_development_goals":[{"display_name":"Industry, innovation and infrastructure","score":0.6100000143051147,"id":"https://metadata.un.org/sdg/9"}],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3196350964.pdf","grobid_xml":"https://content.openalex.org/works/W3196350964.grobid-xml"},"referenced_works_count":47,"referenced_works":["https://openalex.org/W1567503821","https://openalex.org/W1587010832","https://openalex.org/W1943458919","https://openalex.org/W2008479535","https://openalex.org/W2014011458","https://openalex.org/W2028990673","https://openalex.org/W2049461910","https://openalex.org/W2049695835","https://openalex.org/W2064296229","https://openalex.org/W2079698247","https://openalex.org/W2081193615","https://openalex.org/W2091990486","https://openalex.org/W2100414549","https://openalex.org/W2102742419","https://openalex.org/W2121084350","https://openalex.org/W2128782367","https://openalex.org/W2135841285","https://openalex.org/W2135937266","https://openalex.org/W2148646465","https://openalex.org/W2157532207","https://openalex.org/W2164105290","https://openalex.org/W2170204382","https://openalex.org/W2171168235","https://openalex.org/W2298313545","https://openalex.org/W2469179788","https://openalex.org/W2469513535","https://openalex.org/W2517560433","https://openalex.org/W2734519371","https://openalex.org/W2763110557","https://openalex.org/W2777616600","https://openalex.org/W2786944871","https://openalex.org/W2793776506","https://openalex.org/W2809559400","https://openalex.org/W2884023932","https://openalex.org/W2966980041","https://openalex.org/W3111827872","https://openalex.org/W3136164051","https://openalex.org/W3143312578","https://openalex.org/W3161580966","https://openalex.org/W4235786747","https://openalex.org/W4251804709","https://openalex.org/W4254666025","https://openalex.org/W6697318023","https://openalex.org/W6726132807","https://openalex.org/W6748309556","https://openalex.org/W6752429367","https://openalex.org/W6787357462"],"related_works":["https://openalex.org/W3013369813","https://openalex.org/W1995023328","https://openalex.org/W2731612402","https://openalex.org/W2986478207","https://openalex.org/W4312706861","https://openalex.org/W2960013457","https://openalex.org/W1490208943","https://openalex.org/W2599144236","https://openalex.org/W2054751837","https://openalex.org/W2939724256"],"abstract_inverted_index":{"Mutation":[0,22,32,58],"analysis":[1,23,39,141],"assesses":[2],"a":[3,42,55,102,118,151,160,252,296],"test":[4,49,56],"suite\u2019s":[5],"adequacy":[6],"by":[7,247,256],"measuring":[8],"its":[9,249],"ability":[10],"to":[11,51,124,130,137,144,163,202,205,311],"detect":[12],"small":[13],"artificial":[14],"faults,":[15],"systematically":[16],"seeded":[17],"into":[18,303],"the":[19,28,66,89,168,189,209,226,244,270,292,304],"tested":[20],"program.":[21],"is":[24,41,175,309],"considered":[25,63],"one":[26],"of":[27,37,69,81,91,104,108,211,229,275,314],"strongest":[29],"test-adequacy":[30],"criteria.":[31],"testing":[33,43,59,93,126,165,174,298],"builds":[34],"on":[35,117,167,225,261],"top":[36],"mutation":[38,92,125,140,164,173,230,297],"and":[40,83,110,207,215,236,279,284,289,308],"technique":[44],"that":[45,71,199,269,280,300],"uses":[46],"mutants":[47,70,194,198,212,221,235,278],"as":[48,94],"goals":[50],"create":[52],"or":[53],"improve":[54,286],"suite.":[57],"has":[60,87,101],"long":[61],"been":[62],"intractable":[64],"because":[65],"sheer":[67],"number":[68,210],"can":[72],"be":[73,203],"created":[74],"represents":[75,295],"an":[76,95,132],"insurmountable":[77],"problem\u2013both":[78],"in":[79,251],"terms":[80],"human":[82],"computational":[84],"effort.":[85],"This":[86,240],"hindered":[88],"adoption":[90],"industry":[96],"standard.":[97],"For":[98],"example,":[99],"Google":[100],"codebase":[103],"two":[105],"billion":[106],"lines":[107],"code":[109,185,191,217],"more":[111,257,262],"than":[112,188,258,263],"150,000,000":[113],"tests":[114],"are":[115,142,195,200,222],"executed":[116],"daily":[119],"basis.":[120],"The":[121,266],"traditional":[122],"approach":[123,162,246,272,294],"does":[127],"not":[128],"scale":[129],"such":[131,150],"environment;":[133],"even":[134],"existing":[135],"solutions":[136],"speed":[138],"up":[139],"insufficient":[143],"make":[145],"it":[146],"computationally":[147],"feasible":[148],"at":[149],"scale.":[152],"To":[153],"address":[154],"these":[155],"challenges,":[156],"this":[157],"paper":[158,241],"presents":[159],"scalable":[161],"based":[166,224],"following":[169],"main":[170],"ideas:":[171],"(1)":[172],"done":[176],"incrementally,":[177],"mutating":[178],"only":[179],"<italic":[180],"xmlns:mml=\"http://www.w3.org/1998/Math/MathML\"":[181],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">changed":[182],"code</i>":[183],"during":[184],"review,":[186],"rather":[187],"entire":[190],"base;":[192],"(2)":[193],"filtered,":[196],"removing":[197],"likely":[201],"irrelevant":[204,234],"developers,":[206],"limiting":[208],"per":[213,216],"line":[214],"review":[218],"process;":[219],"(3)":[220],"selected":[223],"historical":[227],"performance":[228],"operators,":[231],"further":[232],"eliminating":[233],"improving":[237],"mutant":[238,282,287],"quality.":[239],"empirically":[242],"validates":[243],"proposed":[245,271,293],"analyzing":[248],"effectiveness":[250],"code-review-based":[253],"setting,":[254],"used":[255],"24,000":[259],"developers":[260],"1,000":[264],"projects.":[265],"results":[267],"show":[268],"produces":[273],"orders":[274],"magnitude":[276],"fewer":[277],"context-based":[281],"filtering":[283],"selection":[285],"quality":[288],"actionability.":[290],"Overall,":[291],"framework":[299],"seamlessly":[301],"integrates":[302],"software":[305],"development":[306],"workflow":[307],"applicable":[310],"industrial":[312],"settings":[313],"any":[315],"size.":[316]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":15},{"year":2024,"cited_by_count":22},{"year":2023,"cited_by_count":11},{"year":2022,"cited_by_count":10}],"updated_date":"2026-04-11T08:14:18.477133","created_date":"2025-10-10T00:00:00"}