{"id":"https://openalex.org/W7169577568","doi":"https://doi.org/10.1145/3803437.3805202","title":"Just-in-Time Catching Test Generation at Meta","display_name":"Just-in-Time Catching Test Generation at Meta","publication_year":2026,"publication_date":"2026-07-05","ids":{"openalex":"https://openalex.org/W7169577568","doi":"https://doi.org/10.1145/3803437.3805202"},"language":null,"primary_location":{"id":"doi:10.1145/3803437.3805202","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3803437.3805202","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 34th ACM International Conference on the Foundations of Software Engineering","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3803437.3805202","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5141086622","display_name":"Matthew Becker","orcid":"https://orcid.org/0009-0005-9860-3583"},"institutions":[{"id":"https://openalex.org/I58610484","display_name":"Seattle University","ror":"https://ror.org/02jqc0m91","country_code":"US","type":"education","lineage":["https://openalex.org/I58610484"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Matthew Becker","raw_affiliation_strings":["Meta, Seattle, USA"],"raw_orcid":"https://orcid.org/0009-0005-9860-3583","affiliations":[{"raw_affiliation_string":"Meta, Seattle, USA","institution_ids":["https://openalex.org/I58610484"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5141105140","display_name":"Yifei Chen","orcid":"https://orcid.org/0009-0009-1059-933X"},"institutions":[{"id":"https://openalex.org/I58610484","display_name":"Seattle University","ror":"https://ror.org/02jqc0m91","country_code":"US","type":"education","lineage":["https://openalex.org/I58610484"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yifei Chen","raw_affiliation_strings":["Meta, Seattle, USA"],"raw_orcid":"https://orcid.org/0009-0009-1059-933X","affiliations":[{"raw_affiliation_string":"Meta, Seattle, USA","institution_ids":["https://openalex.org/I58610484"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5141073668","display_name":"Nicholas Cochran","orcid":"https://orcid.org/0009-0001-5212-3817"},"institutions":[{"id":"https://openalex.org/I58610484","display_name":"Seattle University","ror":"https://ror.org/02jqc0m91","country_code":"US","type":"education","lineage":["https://openalex.org/I58610484"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Nicholas Cochran","raw_affiliation_strings":["Meta, Seattle, USA"],"raw_orcid":"https://orcid.org/0009-0001-5212-3817","affiliations":[{"raw_affiliation_string":"Meta, Seattle, USA","institution_ids":["https://openalex.org/I58610484"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5036513368","display_name":"Pouyan Ghasemi","orcid":null},"institutions":[{"id":"https://openalex.org/I58610484","display_name":"Seattle University","ror":"https://ror.org/02jqc0m91","country_code":"US","type":"education","lineage":["https://openalex.org/I58610484"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Pouyan Ghasemi","raw_affiliation_strings":["Meta, Seattle, USA"],"raw_orcid":"https://orcid.org/0009-0008-1685-8067","affiliations":[{"raw_affiliation_string":"Meta, Seattle, USA","institution_ids":["https://openalex.org/I58610484"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5141096447","display_name":"Abhishek Gulati","orcid":"https://orcid.org/0009-0008-5347-5321"},"institutions":[{"id":"https://openalex.org/I58610484","display_name":"Seattle University","ror":"https://ror.org/02jqc0m91","country_code":"US","type":"education","lineage":["https://openalex.org/I58610484"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Abhishek Gulati","raw_affiliation_strings":["Meta, Seattle, USA"],"raw_orcid":"https://orcid.org/0009-0008-5347-5321","affiliations":[{"raw_affiliation_string":"Meta, Seattle, USA","institution_ids":["https://openalex.org/I58610484"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5000019783","display_name":"Mark Harman","orcid":"https://orcid.org/0000-0002-5864-4488"},"institutions":[{"id":"https://openalex.org/I4210111288","display_name":"Meta (United Kingdom)","ror":"https://ror.org/020ye1821","country_code":"GB","type":"company","lineage":["https://openalex.org/I4210111288","https://openalex.org/I4210114444"]},{"id":"https://openalex.org/I45129253","display_name":"University College London","ror":"https://ror.org/02jx3x895","country_code":"GB","type":"education","lineage":["https://openalex.org/I124357947","https://openalex.org/I45129253"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Mark Harman","raw_affiliation_strings":["UCL and Meta, London, United Kingdom"],"raw_orcid":"https://orcid.org/0000-0002-5864-4488","affiliations":[{"raw_affiliation_string":"UCL and Meta, London, United Kingdom","institution_ids":["https://openalex.org/I4210111288","https://openalex.org/I45129253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5124803500","display_name":"Zachary Haluza","orcid":null},"institutions":[{"id":"https://openalex.org/I58610484","display_name":"Seattle University","ror":"https://ror.org/02jqc0m91","country_code":"US","type":"education","lineage":["https://openalex.org/I58610484"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Zachary Haluza","raw_affiliation_strings":["Meta, Seattle, USA"],"raw_orcid":"https://orcid.org/0009-0008-0272-7808","affiliations":[{"raw_affiliation_string":"Meta, Seattle, USA","institution_ids":["https://openalex.org/I58610484"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5141056747","display_name":"Mehrdad Honarkhah","orcid":"https://orcid.org/0009-0006-4172-2124"},"institutions":[{"id":"https://openalex.org/I58610484","display_name":"Seattle University","ror":"https://ror.org/02jqc0m91","country_code":"US","type":"education","lineage":["https://openalex.org/I58610484"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Mehrdad Honarkhah","raw_affiliation_strings":["Meta, Seattle, USA"],"raw_orcid":"https://orcid.org/0009-0006-4172-2124","affiliations":[{"raw_affiliation_string":"Meta, Seattle, USA","institution_ids":["https://openalex.org/I58610484"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5141075459","display_name":"Herve Robert","orcid":"https://orcid.org/0009-0005-9727-4562"},"institutions":[{"id":"https://openalex.org/I58610484","display_name":"Seattle University","ror":"https://ror.org/02jqc0m91","country_code":"US","type":"education","lineage":["https://openalex.org/I58610484"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Herve Robert","raw_affiliation_strings":["Meta, Seattle, USA"],"raw_orcid":"https://orcid.org/0009-0005-9727-4562","affiliations":[{"raw_affiliation_string":"Meta, Seattle, USA","institution_ids":["https://openalex.org/I58610484"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5141095123","display_name":"Jiacheng Liu","orcid":"https://orcid.org/0000-0002-1449-5038"},"institutions":[{"id":"https://openalex.org/I58610484","display_name":"Seattle University","ror":"https://ror.org/02jqc0m91","country_code":"US","type":"education","lineage":["https://openalex.org/I58610484"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jiacheng Liu","raw_affiliation_strings":["Meta, Seattle, USA"],"raw_orcid":"https://orcid.org/0000-0002-1449-5038","affiliations":[{"raw_affiliation_string":"Meta, Seattle, USA","institution_ids":["https://openalex.org/I58610484"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5141109914","display_name":"Weini Liu","orcid":"https://orcid.org/0009-0007-0604-6842"},"institutions":[{"id":"https://openalex.org/I58610484","display_name":"Seattle University","ror":"https://ror.org/02jqc0m91","country_code":"US","type":"education","lineage":["https://openalex.org/I58610484"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Weini Liu","raw_affiliation_strings":["Meta, Seattle, USA"],"raw_orcid":"https://orcid.org/0009-0007-0604-6842","affiliations":[{"raw_affiliation_string":"Meta, Seattle, USA","institution_ids":["https://openalex.org/I58610484"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5003763390","display_name":"Sreeja Thummala","orcid":null},"institutions":[{"id":"https://openalex.org/I58610484","display_name":"Seattle University","ror":"https://ror.org/02jqc0m91","country_code":"US","type":"education","lineage":["https://openalex.org/I58610484"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Sreeja Thummala","raw_affiliation_strings":["Meta, Seattle, USA"],"raw_orcid":"https://orcid.org/0009-0008-4848-5429","affiliations":[{"raw_affiliation_string":"Meta, Seattle, USA","institution_ids":["https://openalex.org/I58610484"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5141080387","display_name":"Xiaoning Yang","orcid":"https://orcid.org/0009-0002-5929-5627"},"institutions":[{"id":"https://openalex.org/I58610484","display_name":"Seattle University","ror":"https://ror.org/02jqc0m91","country_code":"US","type":"education","lineage":["https://openalex.org/I58610484"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Xiaoning Yang","raw_affiliation_strings":["Meta, Seattle, USA"],"raw_orcid":"https://orcid.org/0009-0002-5929-5627","affiliations":[{"raw_affiliation_string":"Meta, Seattle, USA","institution_ids":["https://openalex.org/I58610484"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5141096522","display_name":"Rui Xin","orcid":"https://orcid.org/0009-0003-9563-1242"},"institutions":[{"id":"https://openalex.org/I58610484","display_name":"Seattle University","ror":"https://ror.org/02jqc0m91","country_code":"US","type":"education","lineage":["https://openalex.org/I58610484"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Rui Xin","raw_affiliation_strings":["Meta, Seattle, USA"],"raw_orcid":"https://orcid.org/0009-0003-9563-1242","affiliations":[{"raw_affiliation_string":"Meta, Seattle, USA","institution_ids":["https://openalex.org/I58610484"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5141103399","display_name":"Sophie Zeng","orcid":"https://orcid.org/0009-0003-5012-6699"},"institutions":[{"id":"https://openalex.org/I58610484","display_name":"Seattle University","ror":"https://ror.org/02jqc0m91","country_code":"US","type":"education","lineage":["https://openalex.org/I58610484"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Sophie Zeng","raw_affiliation_strings":["Meta, Seattle, USA"],"raw_orcid":"https://orcid.org/0009-0003-5012-6699","affiliations":[{"raw_affiliation_string":"Meta, Seattle, USA","institution_ids":["https://openalex.org/I58610484"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"282","last_page":"293"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":null,"topics":[],"keywords":[{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.3594000041484833},{"id":"https://openalex.org/keywords/test","display_name":"Test (biology)","score":0.3589000105857849},{"id":"https://openalex.org/keywords/scale","display_name":"Scale (ratio)","score":0.34130001068115234},{"id":"https://openalex.org/keywords/human-error","display_name":"Human error","score":0.3294999897480011},{"id":"https://openalex.org/keywords/statistical-hypothesis-testing","display_name":"Statistical hypothesis testing","score":0.30640000104904175}],"concepts":[{"id":"https://openalex.org/C200601418","wikidata":"https://www.wikidata.org/wiki/Q2193887","display_name":"Reliability engineering","level":1,"score":0.5372999906539917},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.44530001282691956},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.3763999938964844},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.3594000041484833},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.3589000105857849},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.34130001068115234},{"id":"https://openalex.org/C169806903","wikidata":"https://www.wikidata.org/wiki/Q5937752","display_name":"Human error","level":2,"score":0.3294999897480011},{"id":"https://openalex.org/C87007009","wikidata":"https://www.wikidata.org/wiki/Q210832","display_name":"Statistical hypothesis testing","level":2,"score":0.30640000104904175},{"id":"https://openalex.org/C2986587452","wikidata":"https://www.wikidata.org/wiki/Q938438","display_name":"Statistical analysis","level":2,"score":0.29829999804496765},{"id":"https://openalex.org/C151552104","wikidata":"https://www.wikidata.org/wiki/Q7705809","display_name":"Test suite","level":4,"score":0.29319998621940613},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.28870001435279846},{"id":"https://openalex.org/C3019666875","wikidata":"https://www.wikidata.org/wiki/Q455037","display_name":"Assembly line","level":2,"score":0.25519999861717224},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.2547999918460846}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3803437.3805202","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3803437.3805202","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 34th ACM International Conference on the Foundations of Software Engineering","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3803437.3805202","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3803437.3805202","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 34th ACM International Conference on the Foundations of Software Engineering","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":46,"referenced_works":["https://openalex.org/W777621473","https://openalex.org/W1965194038","https://openalex.org/W1971650562","https://openalex.org/W1972091638","https://openalex.org/W1984486655","https://openalex.org/W2035126508","https://openalex.org/W2041713059","https://openalex.org/W2101629181","https://openalex.org/W2105090664","https://openalex.org/W2107147876","https://openalex.org/W2107500604","https://openalex.org/W2115012618","https://openalex.org/W2135841285","https://openalex.org/W2138428785","https://openalex.org/W2147002252","https://openalex.org/W2154236932","https://openalex.org/W2161407365","https://openalex.org/W2161963160","https://openalex.org/W2471601946","https://openalex.org/W2616524285","https://openalex.org/W2808429234","https://openalex.org/W2888728013","https://openalex.org/W2901387825","https://openalex.org/W2964232059","https://openalex.org/W2967869969","https://openalex.org/W3007533873","https://openalex.org/W3023429904","https://openalex.org/W3163304474","https://openalex.org/W3173163365","https://openalex.org/W4210309498","https://openalex.org/W4235786747","https://openalex.org/W4237492309","https://openalex.org/W4242377092","https://openalex.org/W4251543700","https://openalex.org/W4282550244","https://openalex.org/W4299627282","https://openalex.org/W4308643033","https://openalex.org/W4391871252","https://openalex.org/W4392414327","https://openalex.org/W4399214554","https://openalex.org/W4400484392","https://openalex.org/W4400484796","https://openalex.org/W4400484806","https://openalex.org/W4400582872","https://openalex.org/W4412703921","https://openalex.org/W7133239970"],"related_works":[],"abstract_inverted_index":{"We":[0,123],"report":[1],"on":[2],"Just-in-Time":[3,155],"catching":[4,35,156],"test":[5,57],"generation":[6,33,70],"at":[7,32],"Meta,":[8],"designed":[9],"to":[10,39,50,109,128,133,143],"prevent":[11],"bugs":[12,42],"in":[13],"large":[14],"scale":[15],"backend":[16],"systems":[17],"of":[18,20,22,24,138],"hundreds":[19],"millions":[21],"line":[23],"code.":[25],"Unlike":[26],"traditional":[27],"hardening":[28,73],"tests,":[29,62],"which":[30,139],"pass":[31],"time,":[34],"tests":[36,74],"are":[37,107],"meant":[38],"fail,":[40],"surfacing":[41],"before":[43],"code":[44,105],"lands.":[45],"The":[46],"primary":[47],"challenge":[48],"is":[49,157],"reduce":[51,93],"development":[52],"drag":[53],"from":[54,167],"false":[55,83,113],"positive":[56],"failures.":[58],"Analyzing":[59],"22,126":[60],"generated":[61],"we":[63,85],"show":[64,153],"code-change-aware":[65],"methods":[66],"improve":[67],"candidate":[68,126],"catch":[69],"4x":[71],"over":[72,77],"and":[75,88,161],"20x":[76],"coincidentally":[78],"failing":[79],"tests.":[80],"To":[81],"address":[82],"positives,":[84,114,136],"use":[86],"rule-based":[87],"LLM-based":[89],"assessors.":[90],"These":[91],"assessors":[92],"human":[94],"review":[95],"load":[96],"by":[97],"70%.":[98],"Inferential":[99],"statistical":[100],"analysis":[101],"showed":[102],"that":[103,154,162],"human-accepted":[104],"changes":[106,117],"assessed":[108],"have":[110,118,141],"significantly":[111,119],"more":[112,120],"while":[115],"human-rejected":[116],"true":[121,135],"positives.":[122],"reported":[124],"41":[125],"catches":[127],"engineers;":[129],"8":[130],"were":[131],"confirmed":[132],"be":[134],"4":[137],"would":[140],"led":[142],"serious":[144,165],"failures":[145,166],"had":[146],"they":[147],"remained":[148],"uncaught.":[149],"Overall,":[150],"our":[151],"results":[152],"scalable,":[158],"industrially":[159],"applicable,":[160],"it":[163],"prevents":[164],"reaching":[168],"production.":[169]},"counts_by_year":[],"updated_date":"2026-07-18T05:58:53.570196","created_date":"2026-07-18T00:00:00"}