{"id":"https://openalex.org/W2899445138","doi":"https://doi.org/10.1145/3236024.3236057","title":"Testing probabilistic programming systems","display_name":"Testing probabilistic programming systems","publication_year":2018,"publication_date":"2018-10-26","ids":{"openalex":"https://openalex.org/W2899445138","doi":"https://doi.org/10.1145/3236024.3236057","mag":"2899445138"},"language":"en","primary_location":{"id":"doi:10.1145/3236024.3236057","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3236024.3236057","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3236024.3236057","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2018 26th ACM Joint Meeting on European Software Engineering Conference and Symposium on the Foundations of Software Engineering","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3236024.3236057","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5063258857","display_name":"Saikat Dutta","orcid":"https://orcid.org/0000-0001-6021-5407"},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Saikat Dutta","raw_affiliation_strings":["University of Illinois at Urbana-Champaign, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Illinois at Urbana-Champaign, USA","institution_ids":["https://openalex.org/I157725225"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5070516536","display_name":"Owolabi Legunsen","orcid":"https://orcid.org/0000-0001-5631-4816"},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Owolabi Legunsen","raw_affiliation_strings":["University of Illinois at Urbana-Champaign, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Illinois at Urbana-Champaign, USA","institution_ids":["https://openalex.org/I157725225"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5050236609","display_name":"Zixin Huang","orcid":"https://orcid.org/0000-0002-1612-7503"},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Zixin Huang","raw_affiliation_strings":["University of Illinois at Urbana-Champaign, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Illinois at Urbana-Champaign, USA","institution_ids":["https://openalex.org/I157725225"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5057462458","display_name":"Sa\u0161a Misailov\u00edc","orcid":"https://orcid.org/0000-0001-7319-8845"},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Sasa Misailovic","raw_affiliation_strings":["University of Illinois at Urbana-Champaign, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Illinois at Urbana-Champaign, USA","institution_ids":["https://openalex.org/I157725225"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":4.7308,"has_fulltext":true,"cited_by_count":49,"citation_normalized_percentile":{"value":0.9591115,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":97,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"574","last_page":"586"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11303","display_name":"Bayesian Modeling and Causal Inference","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11303","display_name":"Bayesian Modeling and Causal Inference","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9980000257492065,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12423","display_name":"Software Reliability and Analysis Research","score":0.9973999857902527,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8012760877609253},{"id":"https://openalex.org/keywords/probabilistic-logic","display_name":"Probabilistic logic","score":0.7883632183074951},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.6177846789360046},{"id":"https://openalex.org/keywords/extensibility","display_name":"Extensibility","score":0.5619534850120544},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.5442883372306824},{"id":"https://openalex.org/keywords/software-bug","display_name":"Software bug","score":0.42500150203704834},{"id":"https://openalex.org/keywords/fuzz-testing","display_name":"Fuzz testing","score":0.4123501181602478},{"id":"https://openalex.org/keywords/theoretical-computer-science","display_name":"Theoretical computer science","score":0.39864328503608704},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.3795151710510254},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3780944049358368},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.3574954867362976},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.18235939741134644}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8012760877609253},{"id":"https://openalex.org/C49937458","wikidata":"https://www.wikidata.org/wiki/Q2599292","display_name":"Probabilistic logic","level":2,"score":0.7883632183074951},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.6177846789360046},{"id":"https://openalex.org/C32833848","wikidata":"https://www.wikidata.org/wiki/Q4115054","display_name":"Extensibility","level":2,"score":0.5619534850120544},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.5442883372306824},{"id":"https://openalex.org/C1009929","wikidata":"https://www.wikidata.org/wiki/Q179550","display_name":"Software bug","level":3,"score":0.42500150203704834},{"id":"https://openalex.org/C111065885","wikidata":"https://www.wikidata.org/wiki/Q1189053","display_name":"Fuzz testing","level":3,"score":0.4123501181602478},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.39864328503608704},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.3795151710510254},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3780944049358368},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3574954867362976},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.18235939741134644}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3236024.3236057","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3236024.3236057","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3236024.3236057","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2018 26th ACM Joint Meeting on European Software Engineering Conference and Symposium on the Foundations of Software Engineering","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3236024.3236057","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3236024.3236057","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3236024.3236057","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2018 26th ACM Joint Meeting on European Software Engineering Conference and Symposium on the Foundations of Software Engineering","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G4032422751","display_name":null,"funder_award_id":"CCF-1421503","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G4266074940","display_name":null,"funder_award_id":"CCF-1703637,CCF-1629431,CCF-1421503","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G5449669011","display_name":"SHF: Small: Revisiting Assumptions of Regression Testing","funder_award_id":"1421503","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G5844732816","display_name":"XPS: FULL: Breaking the Scalability Wall of Shared Memory through Fast On-Chip Wireless Communication","funder_award_id":"1629431","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G7469296022","display_name":"SHF: Medium: Collaborative Research: Testing in the Era of Approximation","funder_award_id":"1703637","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2899445138.pdf","grobid_xml":"https://content.openalex.org/works/W2899445138.grobid-xml"},"referenced_works_count":86,"referenced_works":["https://openalex.org/W109452506","https://openalex.org/W156498718","https://openalex.org/W157156687","https://openalex.org/W634934296","https://openalex.org/W814172419","https://openalex.org/W1502951647","https://openalex.org/W1519683776","https://openalex.org/W1531203382","https://openalex.org/W1556520454","https://openalex.org/W1589239973","https://openalex.org/W1688932025","https://openalex.org/W1702158037","https://openalex.org/W1844453107","https://openalex.org/W1890754682","https://openalex.org/W1966021031","https://openalex.org/W1980452149","https://openalex.org/W1983878972","https://openalex.org/W1988524530","https://openalex.org/W1989703131","https://openalex.org/W2000974833","https://openalex.org/W2002934700","https://openalex.org/W2006599089","https://openalex.org/W2008626182","https://openalex.org/W2016716561","https://openalex.org/W2025704776","https://openalex.org/W2030405312","https://openalex.org/W2041650849","https://openalex.org/W2048025009","https://openalex.org/W2049639527","https://openalex.org/W2060573639","https://openalex.org/W2065948900","https://openalex.org/W2071952624","https://openalex.org/W2081368694","https://openalex.org/W2095445208","https://openalex.org/W2098456636","https://openalex.org/W2098681203","https://openalex.org/W2102360969","https://openalex.org/W2103072691","https://openalex.org/W2103201337","https://openalex.org/W2112820511","https://openalex.org/W2118155805","https://openalex.org/W2118577607","https://openalex.org/W2119342939","https://openalex.org/W2129164226","https://openalex.org/W2130514924","https://openalex.org/W2134124281","https://openalex.org/W2135394066","https://openalex.org/W2136650506","https://openalex.org/W2141670510","https://openalex.org/W2146376107","https://openalex.org/W2151704521","https://openalex.org/W2155049725","https://openalex.org/W2159933174","https://openalex.org/W2160829641","https://openalex.org/W2162036145","https://openalex.org/W2170737051","https://openalex.org/W2295266283","https://openalex.org/W2324595780","https://openalex.org/W2390518826","https://openalex.org/W2395880299","https://openalex.org/W2406252837","https://openalex.org/W2409355369","https://openalex.org/W2414287720","https://openalex.org/W2494200839","https://openalex.org/W2530895108","https://openalex.org/W2539792571","https://openalex.org/W2565778127","https://openalex.org/W2574963492","https://openalex.org/W2577537660","https://openalex.org/W2617813225","https://openalex.org/W2620081107","https://openalex.org/W2623327568","https://openalex.org/W2758443906","https://openalex.org/W2767209415","https://openalex.org/W2767872212","https://openalex.org/W2795266985","https://openalex.org/W2806676720","https://openalex.org/W2921051283","https://openalex.org/W2950204717","https://openalex.org/W2963702270","https://openalex.org/W2964064835","https://openalex.org/W3020947624","https://openalex.org/W3089425725","https://openalex.org/W4238083723","https://openalex.org/W4238858416","https://openalex.org/W4243718993"],"related_works":["https://openalex.org/W614438062","https://openalex.org/W2163814182","https://openalex.org/W3173990398","https://openalex.org/W4381785649","https://openalex.org/W4205454537","https://openalex.org/W4226494072","https://openalex.org/W4385301282","https://openalex.org/W3170526652","https://openalex.org/W3105061167","https://openalex.org/W4287849816"],"abstract_inverted_index":{"Probabilistic":[0],"programming":[1,24],"systems":[2,25,39,51],"(PP":[3],"systems)":[4],"allow":[5],"developers":[6],"to":[7,93,114,172],"model":[8],"stochastic":[9],"phenomena":[10],"and":[11,20,40,76,106,177,182],"perform":[12,58],"efficient":[13],"inference":[14],"on":[15],"the":[16,121,133,137,173],"models.":[17],"The":[18],"number":[19],"adoption":[21],"of":[22,35,96,123,159],"probabilistic":[23,97,104],"is":[26,31,52],"growing":[27],"significantly.":[28],"However,":[29],"there":[30],"no":[32,41],"prior":[33],"study":[34],"bugs":[36,69,130,155],"in":[37,70,156],"these":[38,116,160],"methodology":[42],"for":[43,84,108],"systematically":[44],"testing":[45,49,85],"PP":[46,50,73,86,125,161,175],"systems.":[47,87,162],"Yet,":[48],"highly":[53],"non-trivial,":[54],"especially":[55],"when":[56],"they":[57],"approximate":[59],"inference.":[60],"In":[61],"this":[62],"paper,":[63],"we":[64,150,170],"characterize":[65],"118":[66],"previously":[67,153],"reported":[68],"three":[71,174],"open-source":[72],"systems\u2014Edward,":[74],"Pyro":[75],"Stan\u2014and":[77],"pro-":[78],"pose":[79],"ProbFuzz,":[80,149],"an":[81,144],"extensible":[82],"system":[83],"Prob-":[88],"Fuzz":[89],"allows":[90],"a":[91],"developer":[92],"specify":[94],"templates":[95],"models,":[98],"from":[99,135],"which":[100,119],"it":[101],"generates":[102],"concrete":[103,117],"programs":[105,139],"data":[107],"testing.":[109],"ProbFuzz":[110,127],"uses":[111],"language-specific":[112],"translators":[113],"generate":[115],"programs,":[118],"use":[120],"APIs":[122],"each":[124],"system.":[126],"finds":[128],"potential":[129],"by":[131],"checking":[132],"output":[134],"running":[136],"generated":[138],"against":[140],"several":[141],"oracles,":[142],"including":[143],"accu-":[145],"racy":[146],"checker.":[147],"Using":[148],"found":[151],"67":[152],"unknown":[154],"recent":[157],"versions":[158],"Developers":[163],"already":[164],"accepted":[165],"51":[166],"bug":[167],"fixes":[168],"that":[169],"submitted":[171],"systems,":[176,180],"their":[178],"underlying":[179],"PyTorch":[181],"TensorFlow.":[183]},"counts_by_year":[{"year":2024,"cited_by_count":4},{"year":2023,"cited_by_count":6},{"year":2022,"cited_by_count":11},{"year":2021,"cited_by_count":12},{"year":2020,"cited_by_count":11},{"year":2019,"cited_by_count":5}],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-10-10T00:00:00"}