{"id":"https://openalex.org/W7221541982","doi":"https://doi.org/10.48550/arxiv.2610.12544","title":"Transformer-Based Masked Diffusion Models Provably Learn Distributions with Latent Structure","display_name":"Transformer-Based Masked Diffusion Models Provably Learn Distributions with Latent Structure","publication_year":2026,"publication_date":"2026-10-08","ids":{"openalex":"https://openalex.org/W7221541982","doi":"https://doi.org/10.48550/arxiv.2610.12544"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2610.12544","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2610.12544","pdf_url":"https://arxiv.org/pdf/2610.12544","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2610.12544","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5044215622","display_name":"Jingda Wu","orcid":"https://orcid.org/0000-0002-7336-4492","observed_orcids":["https://orcid.org/0000-0002-7336-4492"]},"institutions":[{"id":"https://openalex.org/I27837315","display_name":"University of Michigan","ror":"https://ror.org/00jmfr291","country_code":"US","type":"education","lineage":["https://openalex.org/I27837315"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Wu, Jingda","raw_affiliation_strings":["Department of Industrial and Operations Engineering , University of Michigan , Ann Arbor , USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Industrial and Operations Engineering , University of Michigan , Ann Arbor , USA","institution_ids":["https://openalex.org/I27837315"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5080132489","display_name":"Changxiao Cai","orcid":"https://orcid.org/0000-0003-4941-7627","observed_orcids":["https://orcid.org/0000-0003-4941-7627"]},"institutions":[{"id":"https://openalex.org/I27837315","display_name":"University of Michigan","ror":"https://ror.org/00jmfr291","country_code":"US","type":"education","lineage":["https://openalex.org/I27837315"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Cai, Changxiao","raw_affiliation_strings":["Department of Industrial and Operations Engineering , University of Michigan , Ann Arbor , USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Industrial and Operations Engineering , University of Michigan , Ann Arbor , USA","institution_ids":["https://openalex.org/I27837315"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I27837315"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.5795999765396118,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.5795999765396118,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10775","display_name":"Generative Adversarial Networks and Image Synthesis","score":0.16200000047683716,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.07620000094175339,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/masked-diffusion-models","display_name":"masked diffusion models","score":1.0},{"id":"https://openalex.org/keywords/discrete-data-generation","display_name":"discrete data generation","score":0.6930000185966492},{"id":"https://openalex.org/keywords/sample-complexity","display_name":"sample complexity","score":0.6110000014305115},{"id":"https://openalex.org/keywords/statistical-guarantees","display_name":"statistical guarantees","score":0.6710000038146973},{"id":"https://openalex.org/keywords/hidden-markov-tree-model","display_name":"hidden Markov tree","score":0.7549999952316284},{"id":"https://openalex.org/keywords/mixture-models","display_name":"mixture models","score":0.6899999976158142},{"id":"https://openalex.org/keywords/kullback-leibler-divergence","display_name":"Kullback-Leibler divergence","score":0.5630000233650208},{"id":"https://openalex.org/keywords/latent-structure","display_name":"latent structure","score":0.6100000143051147},{"id":"https://openalex.org/keywords/transformer","display_name":"transformer models","score":0.41499999165534973},{"id":"https://openalex.org/keywords/curse-of-dimensionality","display_name":"curse of dimensionality","score":0.6610000133514404}],"concepts":[{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.49559998512268066},{"id":"https://openalex.org/C98763669","wikidata":"https://www.wikidata.org/wiki/Q176645","display_name":"Markov chain","level":2,"score":0.474700003862381},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.46369999647140503},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4569000005722046},{"id":"https://openalex.org/C78469957","wikidata":"https://www.wikidata.org/wiki/Q838062","display_name":"occam","level":2,"score":0.4180000126361847},{"id":"https://openalex.org/C61224824","wikidata":"https://www.wikidata.org/wiki/Q2260434","display_name":"Mixture model","level":2,"score":0.39739999175071716},{"id":"https://openalex.org/C64939953","wikidata":"https://www.wikidata.org/wiki/Q3859882","display_name":"Hidden semi-Markov model","level":5,"score":0.38960000872612},{"id":"https://openalex.org/C23224414","wikidata":"https://www.wikidata.org/wiki/Q176769","display_name":"Hidden Markov model","level":2,"score":0.38769999146461487},{"id":"https://openalex.org/C140779682","wikidata":"https://www.wikidata.org/wiki/Q210868","display_name":"Sampling (signal processing)","level":3,"score":0.3831999897956848},{"id":"https://openalex.org/C111030470","wikidata":"https://www.wikidata.org/wiki/Q1430460","display_name":"Curse of dimensionality","level":2,"score":0.38190001249313354},{"id":"https://openalex.org/C114289077","wikidata":"https://www.wikidata.org/wiki/Q3284399","display_name":"Statistical model","level":2,"score":0.3747999966144562},{"id":"https://openalex.org/C149441793","wikidata":"https://www.wikidata.org/wiki/Q200726","display_name":"Probability distribution","level":2,"score":0.361299991607666},{"id":"https://openalex.org/C159886148","wikidata":"https://www.wikidata.org/wiki/Q176645","display_name":"Markov process","level":2,"score":0.35580000281333923},{"id":"https://openalex.org/C107673813","wikidata":"https://www.wikidata.org/wiki/Q812534","display_name":"Bayesian probability","level":2,"score":0.3425000011920929},{"id":"https://openalex.org/C163836022","wikidata":"https://www.wikidata.org/wiki/Q6771326","display_name":"Markov model","level":3,"score":0.33570000529289246},{"id":"https://openalex.org/C207390915","wikidata":"https://www.wikidata.org/wiki/Q1230525","display_name":"Divergence (linguistics)","level":2,"score":0.33489999175071716},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.33079999685287476},{"id":"https://openalex.org/C168167062","wikidata":"https://www.wikidata.org/wiki/Q1117970","display_name":"Component (thermodynamics)","level":2,"score":0.3264000117778778},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.323199987411499},{"id":"https://openalex.org/C160920958","wikidata":"https://www.wikidata.org/wiki/Q7662746","display_name":"Synthetic data","level":2,"score":0.3231000006198883},{"id":"https://openalex.org/C2777601683","wikidata":"https://www.wikidata.org/wiki/Q6499736","display_name":"Vocabulary","level":2,"score":0.3174000084400177},{"id":"https://openalex.org/C56672385","wikidata":"https://www.wikidata.org/wiki/Q17157111","display_name":"Mixture distribution","level":3,"score":0.30869999527931213},{"id":"https://openalex.org/C199845137","wikidata":"https://www.wikidata.org/wiki/Q145490","display_name":"Network topology","level":2,"score":0.3082999885082245},{"id":"https://openalex.org/C111350023","wikidata":"https://www.wikidata.org/wiki/Q1191869","display_name":"Markov chain Monte Carlo","level":3,"score":0.30469998717308044},{"id":"https://openalex.org/C163797641","wikidata":"https://www.wikidata.org/wiki/Q2067937","display_name":"Tree structure","level":3,"score":0.2987000048160553},{"id":"https://openalex.org/C61067352","wikidata":"https://www.wikidata.org/wiki/Q1897429","display_name":"Ultrametric space","level":3,"score":0.2946000099182129},{"id":"https://openalex.org/C54907487","wikidata":"https://www.wikidata.org/wiki/Q7915688","display_name":"Variable-order Markov model","level":4,"score":0.2913999855518341},{"id":"https://openalex.org/C90119067","wikidata":"https://www.wikidata.org/wiki/Q43260","display_name":"Polynomial","level":2,"score":0.28940001130104065},{"id":"https://openalex.org/C87465248","wikidata":"https://www.wikidata.org/wiki/Q1417790","display_name":"Minimum description length","level":2,"score":0.2854999899864197},{"id":"https://openalex.org/C113174947","wikidata":"https://www.wikidata.org/wiki/Q2859736","display_name":"Tree (set theory)","level":2,"score":0.2849000096321106},{"id":"https://openalex.org/C311688","wikidata":"https://www.wikidata.org/wiki/Q2393193","display_name":"Time complexity","level":2,"score":0.28439998626708984},{"id":"https://openalex.org/C162319229","wikidata":"https://www.wikidata.org/wiki/Q175263","display_name":"Data structure","level":2,"score":0.28380000591278076},{"id":"https://openalex.org/C65965080","wikidata":"https://www.wikidata.org/wiki/Q1806885","display_name":"Latent variable model","level":3,"score":0.2833000123500824},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.2824000120162964},{"id":"https://openalex.org/C177769412","wikidata":"https://www.wikidata.org/wiki/Q278090","display_name":"Prior probability","level":3,"score":0.2809000015258789},{"id":"https://openalex.org/C129848803","wikidata":"https://www.wikidata.org/wiki/Q2564360","display_name":"Sample size determination","level":2,"score":0.2791999876499176},{"id":"https://openalex.org/C167723999","wikidata":"https://www.wikidata.org/wiki/Q3773214","display_name":"Sampling distribution","level":2,"score":0.2718999981880188},{"id":"https://openalex.org/C51167844","wikidata":"https://www.wikidata.org/wiki/Q4422623","display_name":"Latent variable","level":2,"score":0.26759999990463257},{"id":"https://openalex.org/C158424031","wikidata":"https://www.wikidata.org/wiki/Q1191905","display_name":"Gibbs sampling","level":3,"score":0.2653000056743622},{"id":"https://openalex.org/C198531522","wikidata":"https://www.wikidata.org/wiki/Q485146","display_name":"Sample (material)","level":2,"score":0.26429998874664307}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2610.12544","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2610.12544","pdf_url":"https://arxiv.org/pdf/2610.12544","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2610.12544","doi":"https://doi.org/10.48550/arxiv.2610.12544","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2610.12544","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2610.12544","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2610.12544","pdf_url":"https://arxiv.org/pdf/2610.12544","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G3210862149","display_name":"Robust Data-Driven Decision-Making: Human-AI Alignment, Adaptivity, and Optimality","funder_award_id":"2515333","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7221541982.pdf","grobid_xml":"https://content.openalex.org/works/W7221541982.grobid-xml"},"referenced_works_count":23,"referenced_works":["https://openalex.org/W2908510526","https://openalex.org/W3110257065","https://openalex.org/W4296957262","https://openalex.org/W4387964173","https://openalex.org/W4403586429","https://openalex.org/W4403621327","https://openalex.org/W4403799408","https://openalex.org/W4414846785","https://openalex.org/W4414970539","https://openalex.org/W4415098679","https://openalex.org/W4416027183","https://openalex.org/W4416138639","https://openalex.org/W7090709548","https://openalex.org/W7102754076","https://openalex.org/W7129339926","https://openalex.org/W7140197266","https://openalex.org/W7162762430","https://openalex.org/W7164527974","https://openalex.org/W7203478542","https://openalex.org/W7204147297","https://openalex.org/W7204176442","https://openalex.org/W7213577207","https://openalex.org/W7213937302"],"related_works":["https://openalex.org/W7133345451","https://openalex.org/W4416027183","https://openalex.org/W4414858786","https://openalex.org/W4416256030","https://openalex.org/W4403751234","https://openalex.org/W7160907622","https://openalex.org/W7162430223","https://openalex.org/W7127408065","https://openalex.org/W7131076213","https://openalex.org/W7133204917"],"abstract_inverted_index":{"Masked":[0],"diffusion":[1],"models":[2],"(MDMs)":[3],"generate":[4],"discrete":[5],"data":[6,120],"by":[7,93,125],"iteratively":[8],"unmasking":[9,134],"multiple":[10],"tokens":[11],"in":[12,139,150],"parallel":[13,28,133],"using":[14],"a":[15,62,89,127,129],"learned":[16,37],"mask":[17,35],"predictor.":[18],"Their":[19],"output":[20],"carries":[21],"two":[22],"errors:":[23],"parallelization":[24],"error":[25,32],"due":[26],"to":[27,182],"sampling,":[29],"and":[30,47,114,158,167],"estimation":[31],"from":[33,68],"the":[34,48,55,69,119,153,159,174,183,195],"predictor":[36],"with":[38,54,88,132,142],"finite":[39],"samples.":[40],"Existing":[41],"theory":[42],"treats":[43],"these":[44],"errors":[45],"separately,":[46],"available":[49],"statistical":[50,79],"bounds":[51],"typically":[52],"scale":[53],"number":[56,154,160],"$V^L$":[57],"of":[58,64,71,104,155,161,173,198],"length-$L$":[59],"sequences":[60],"over":[61],"vocabulary":[63],"size":[65],"$V$,":[66,152],"suffering":[67],"curse":[70],"dimensionality.":[72],"In":[73],"this":[74,100],"work,":[75],"we":[76,98],"establish":[77],"end-to-end":[78],"guarantees":[80],"showing":[81],"that":[82],"transformer-based":[83,130],"MDMs":[84,199],"learn":[85],"structured":[86],"distributions":[87],"sample":[90],"complexity":[91],"governed":[92],"their":[94],"latent":[95,110,185],"structure.":[96],"Specifically,":[97],"model":[99],"structure":[101,186],"through":[102],"mixtures":[103],"hidden":[105,156],"Markov":[106],"trees,":[107],"which":[108],"capture":[109],"dependencies":[111],"within":[112],"components":[113],"heterogeneity":[115],"across":[116],"components.":[117,163],"When":[118],"distribution":[121],"is":[122,148],"well":[123],"approximated":[124],"such":[126],"mixture,":[128],"MDM":[131],"schedules":[135],"achieves":[136],"$\\varepsilon$-accurate":[137],"sampling":[138,168],"KL":[140],"divergence":[141],"$\\widetilde{O}(LK_{\\mathrm{par}}/\\varepsilon)$":[143],"training":[144],"samples,":[145],"where":[146],"$K_{\\mathrm{par}}$":[147],"polynomial":[149],"$L$,":[151],"states,":[157],"mixture":[162],"The":[164],"transformer":[165],"network":[166],"procedure":[169],"require":[170],"no":[171],"knowledge":[172],"tree":[175],"topologies":[176],"or":[177],"component":[178],"parameters,":[179],"thereby":[180],"adapting":[181],"unknown":[184],"automatically.":[187],"Overall,":[188],"our":[189],"results":[190],"provide":[191],"theoretical":[192],"justification":[193],"for":[194],"empirical":[196],"effectiveness":[197],"on":[200],"real-world":[201],"data.":[202]},"counts_by_year":[],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2026-10-13T00:00:00"}