{"id":"https://openalex.org/W7166889437","doi":"https://doi.org/10.48550/arxiv.2606.31630","title":"Calibration, Not Compilation: Detecting and Repairing Misspecified Probabilistic Programs Written by Language Models","display_name":"Calibration, Not Compilation: Detecting and Repairing Misspecified Probabilistic Programs Written by Language Models","publication_year":2026,"publication_date":"2026-06-30","ids":{"openalex":"https://openalex.org/W7166889437","doi":"https://doi.org/10.48550/arxiv.2606.31630"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.31630","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.31630","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.31630","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5139749451","display_name":"Jian Xu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xu, Jian","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139830213","display_name":"Delu Zeng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zeng, Delu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139732996","display_name":"John Paisley","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Paisley, John","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5139754912","display_name":"Qibin Zhao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhao, Qibin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.25769999623298645,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.25769999623298645,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.09380000084638596,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12127","display_name":"Software System Performance and Reliability","score":0.065700002014637,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/probabilistic-logic","display_name":"Probabilistic logic","score":0.6575000286102295},{"id":"https://openalex.org/keywords/oracle","display_name":"Oracle","score":0.6190000176429749},{"id":"https://openalex.org/keywords/calibration","display_name":"Calibration","score":0.5551000237464905},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.5375000238418579},{"id":"https://openalex.org/keywords/correctness","display_name":"Correctness","score":0.5044999718666077},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.5012000203132629},{"id":"https://openalex.org/keywords/test-suite","display_name":"Test suite","score":0.4869999885559082},{"id":"https://openalex.org/keywords/bayesian-probability","display_name":"Bayesian probability","score":0.44699999690055847},{"id":"https://openalex.org/keywords/test-case","display_name":"Test case","score":0.42399999499320984},{"id":"https://openalex.org/keywords/posterior-probability","display_name":"Posterior probability","score":0.41850000619888306}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7870000004768372},{"id":"https://openalex.org/C49937458","wikidata":"https://www.wikidata.org/wiki/Q2599292","display_name":"Probabilistic logic","level":2,"score":0.6575000286102295},{"id":"https://openalex.org/C55166926","wikidata":"https://www.wikidata.org/wiki/Q2892946","display_name":"Oracle","level":2,"score":0.6190000176429749},{"id":"https://openalex.org/C165838908","wikidata":"https://www.wikidata.org/wiki/Q736777","display_name":"Calibration","level":2,"score":0.5551000237464905},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.5375000238418579},{"id":"https://openalex.org/C55439883","wikidata":"https://www.wikidata.org/wiki/Q360812","display_name":"Correctness","level":2,"score":0.5044999718666077},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.5012000203132629},{"id":"https://openalex.org/C151552104","wikidata":"https://www.wikidata.org/wiki/Q7705809","display_name":"Test suite","level":4,"score":0.4869999885559082},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.46959999203681946},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.45320001244544983},{"id":"https://openalex.org/C107673813","wikidata":"https://www.wikidata.org/wiki/Q812534","display_name":"Bayesian probability","level":2,"score":0.44699999690055847},{"id":"https://openalex.org/C128942645","wikidata":"https://www.wikidata.org/wiki/Q1568346","display_name":"Test case","level":3,"score":0.42399999499320984},{"id":"https://openalex.org/C57830394","wikidata":"https://www.wikidata.org/wiki/Q278079","display_name":"Posterior probability","level":3,"score":0.41850000619888306},{"id":"https://openalex.org/C917703","wikidata":"https://www.wikidata.org/wiki/Q7239668","display_name":"Predictive inference","level":5,"score":0.4066999852657318},{"id":"https://openalex.org/C23224414","wikidata":"https://www.wikidata.org/wiki/Q176769","display_name":"Hidden Markov model","level":2,"score":0.3873000144958496},{"id":"https://openalex.org/C196083921","wikidata":"https://www.wikidata.org/wiki/Q7915758","display_name":"Variance (accounting)","level":2,"score":0.38429999351501465},{"id":"https://openalex.org/C114289077","wikidata":"https://www.wikidata.org/wiki/Q3284399","display_name":"Statistical model","level":2,"score":0.37880000472068787},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.37229999899864197},{"id":"https://openalex.org/C61326573","wikidata":"https://www.wikidata.org/wiki/Q1496376","display_name":"Gaussian process","level":3,"score":0.36640000343322754},{"id":"https://openalex.org/C70437156","wikidata":"https://www.wikidata.org/wiki/Q7228652","display_name":"Pooling","level":2,"score":0.35120001435279846},{"id":"https://openalex.org/C2776036281","wikidata":"https://www.wikidata.org/wiki/Q48769818","display_name":"Constraint (computer-aided design)","level":2,"score":0.34360000491142273},{"id":"https://openalex.org/C79581498","wikidata":"https://www.wikidata.org/wiki/Q1367530","display_name":"Suite","level":2,"score":0.33730000257492065},{"id":"https://openalex.org/C80478641","wikidata":"https://www.wikidata.org/wiki/Q195771","display_name":"Sequential analysis","level":2,"score":0.33649998903274536},{"id":"https://openalex.org/C160234255","wikidata":"https://www.wikidata.org/wiki/Q812535","display_name":"Bayesian inference","level":3,"score":0.31520000100135803},{"id":"https://openalex.org/C177212765","wikidata":"https://www.wikidata.org/wiki/Q627335","display_name":"Workflow","level":2,"score":0.3111000061035156},{"id":"https://openalex.org/C87007009","wikidata":"https://www.wikidata.org/wiki/Q210832","display_name":"Statistical hypothesis testing","level":2,"score":0.3091999888420105},{"id":"https://openalex.org/C183115368","wikidata":"https://www.wikidata.org/wiki/Q856577","display_name":"Weighting","level":2,"score":0.30790001153945923},{"id":"https://openalex.org/C40422974","wikidata":"https://www.wikidata.org/wiki/Q741248","display_name":"Assertion","level":2,"score":0.30720001459121704},{"id":"https://openalex.org/C100906024","wikidata":"https://www.wikidata.org/wiki/Q205692","display_name":"Poisson distribution","level":2,"score":0.3012000024318695},{"id":"https://openalex.org/C148027188","wikidata":"https://www.wikidata.org/wiki/Q907375","display_name":"Unit testing","level":3,"score":0.2924000024795532},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.2903999984264374},{"id":"https://openalex.org/C2779714256","wikidata":"https://www.wikidata.org/wiki/Q25305062","display_name":"Multiple Models","level":2,"score":0.27900001406669617},{"id":"https://openalex.org/C2777548347","wikidata":"https://www.wikidata.org/wiki/Q5456937","display_name":"Flagging","level":2,"score":0.273499995470047},{"id":"https://openalex.org/C2780069185","wikidata":"https://www.wikidata.org/wiki/Q7977945","display_name":"Equivalence (formal languages)","level":2,"score":0.27090001106262207},{"id":"https://openalex.org/C94966114","wikidata":"https://www.wikidata.org/wiki/Q29256","display_name":"Black box","level":2,"score":0.2533000111579895}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.31630","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.31630","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.31630","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.31630","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Language":[0],"models":[1,191],"increasingly":[2],"write":[3,203],"probabilistic":[4,242],"programs":[5,201],"(in":[6],"NumPyro,":[7],"Stan,":[8],"or":[9,43],"Pyro),":[10],"but":[11,56],"a":[12,28,34,44,53,86,120,135,143,174,227],"program":[13,128],"that":[14,124,180],"compiles,":[15],"runs,":[16],"and":[17,72,119,184,220,230],"passes":[18],"every":[19],"unit":[20],"test":[21,54,176],"can":[22],"still":[23],"be":[24],"\\emph{statistically}":[25],"wrong":[26],"--":[27,118,163,183],"Gaussian":[29],"likelihood":[30],"for":[31,36,142,206,241],"heavy-tailed":[32],"data,":[33],"Poisson":[35],"over-dispersed":[37],"counts,":[38],"an":[39,115,151],"invalid":[40],"prior":[41],"support,":[42],"pathological":[45],"parameterization.":[46],"The":[47],"right":[48],"verifier":[49],"is":[50,165,238,245],"therefore":[51],"not":[52,247],"suite":[55],"the":[57,100,111,236,239],"Bayesian":[58],"workflow":[59],"itself:":[60],"posterior":[61],"predictive":[62,74],"checks,":[63],"simulation-based":[64],"calibration,":[65,246],"sampler":[66],"diagnostics":[67],"($\\hat":[68],"R$,":[69],"divergences,":[70],"ESS),":[71],"held-out":[73],"density.":[75],"We":[76],"study":[77],"this":[78],"calibration":[79,158],"oracle":[80],"along":[81],"three":[82],"axes.":[83],"\\textbf{Detection:}":[84],"on":[85,189,200],"benchmark":[87],"of":[88,210],"$14$":[89],"misspecification":[90],"types":[91],"across":[92,155],"$10$":[93],"model":[94,138],"families":[95],"($200$":[96],"instances),":[97],"it":[98],"flags":[99],"bug":[101],"with":[102],"AUC":[103],"$0.97$":[104],"($88\\%$":[105],"at":[106,172],"$2\\%$":[107],"FPR":[108],"\\emph{when":[109],"handed":[110],"correct":[112,127],"reference":[113],"program,":[114],"upper":[116,132],"bound})":[117],"fully":[121],"\\emph{reference-free}":[122],"version":[123],"uses":[125],"no":[126,170,187],"reaches":[129],"$62$--$78\\%$":[130],"(the":[131],"figure":[133],"from":[134,204],"small":[136],"automated":[137],"search),":[139],"versus":[140],"$0\\%$":[141],"unit-test":[144,161],"oracle.":[145],"\\textbf{Repair:}":[146],"used":[147],"as":[148],"feedback":[149,162,171,188],"in":[150],"LLM":[152],"repair":[153,182,222],"loop":[154],"fifteen":[156],"models,":[157],"significantly":[159,223],"outperforms":[160],"which":[164],"itself":[166],"\\emph{significantly":[167],"worse":[168],"than":[169],"all},":[173],"passing":[175],"inducing":[177],"false":[178],"confidence":[179],"suppresses":[181],"improves":[185],"over":[186],"strong-but-unsaturated":[190],"(GPT-5.1":[192],"$33{\\to}92\\%$,":[193],"Claude":[194],"$75{\\to}100\\%$;":[195],"paired":[196],"McNemar,":[197],"$n{=}228$).":[198],"\\textbf{Reality:}":[199],"LLMs":[202],"scratch":[205],"neutral":[207],"briefs,":[208],"$15$--$47\\%$":[209],"runnable":[211],"ones":[212],"are":[213],"statistically":[214],"misspecified":[215],"(unit":[216],"tests":[217],"catch":[218],"none),":[219],"calibration-guided":[221],"beats":[224],"LLM-as-judge":[225],"review,":[226],"Bayesian-workflow":[228],"checklist,":[229],"data-summary":[231],"self-debug.":[232],"Across":[233],"all":[234],"three,":[235],"lesson":[237],"same:":[240],"programs,":[243],"correctness":[244],"compilation.":[248]},"counts_by_year":[],"updated_date":"2026-07-02T06:18:51.028212","created_date":"2026-07-02T00:00:00"}