{"id":"https://openalex.org/W2096684396","doi":"https://doi.org/10.1162/003355304772839588","title":"How Much Should We Trust Differences-In-Differences Estimates?","display_name":"How Much Should We Trust Differences-In-Differences Estimates?","alternate_titles":[],"publication_year":2004,"publication_date":"2004-02-01","ids":{"openalex":"https://openalex.org/W2096684396","doi":"https://doi.org/10.1162/003355304772839588","mag":"2096684396"},"language":"en","primary_location":{"id":"doi:10.1162/003355304772839588","doi":"https://doi.org/10.1162/003355304772839588","is_oa":false,"landing_page_url":"https://doi.org/10.1162/003355304772839588","pdf_url":null,"source":{"id":"https://openalex.org/S203860005","display_name":"The Quarterly Journal of Economics","issn_l":"0033-5533","issn":["0033-5533","1531-4650"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["abdc-a-star","anvur-class-a","anvur-scientific","cwts-core","ft50","jufo-3","norway-2","poland-200","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310311648","host_organization_name":"Oxford University Press","host_organization_lineage":["https://openalex.org/P4310311648","https://openalex.org/P4310311647"],"host_organization_lineage_names":["Oxford University Press","University of Oxford"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"The Quarterly Journal of Economics","raw_type":"journal-article"},"parent_works":[],"lineage":["https://openalex.org/W2096684396"],"child_works_count":0,"type":"article","is_traditional":true,"indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"http://hdl.handle.net/1721.1/63690","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"M. Bertrand","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I1321305853","display_name":"National Bureau of Economic Research","ror":"https://ror.org/04grmx538","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I1321305853"]},{"id":"https://openalex.org/I40347166","display_name":"University of Chicago","ror":"https://ror.org/024mw5h28","country_code":"US","type":"education","lineage":["https://openalex.org/I40347166"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"M. Bertrand","raw_affiliation_strings":["University of Chicago Graduate School of Business, National Bureau of Economic Research, and Center for Economic Policy Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Chicago Graduate School of Business, National Bureau of Economic Research, and Center for Economic Policy Research","institution_ids":["https://openalex.org/I1321305853","https://openalex.org/I40347166"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5015565298","display_name":"Esther Duflo","orcid":"https://orcid.org/0000-0001-6105-617X","observed_orcids":["https://orcid.org/0000-0001-6105-617X"]},"institutions":[{"id":"https://openalex.org/I1321305853","display_name":"National Bureau of Economic Research","ror":"https://ror.org/04grmx538","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I1321305853"]},{"id":"https://openalex.org/I63966007","display_name":"Massachusetts Institute of Technology","ror":"https://ror.org/042nb2s44","country_code":"US","type":"education","lineage":["https://openalex.org/I63966007"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"E. Duflo","raw_affiliation_strings":["Massachusetts Institute of Technology, National Bureau of Economic Research, and Center for Economic Policy Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Massachusetts Institute of Technology, National Bureau of Economic Research, and Center for Economic Policy Research","institution_ids":["https://openalex.org/I1321305853","https://openalex.org/I63966007"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5034703281","display_name":"Sendhil Mullainathan","orcid":"https://orcid.org/0000-0001-8508-4052","observed_orcids":["https://orcid.org/0000-0001-8508-4052"]},"institutions":[{"id":"https://openalex.org/I1321305853","display_name":"National Bureau of Economic Research","ror":"https://ror.org/04grmx538","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I1321305853"]},{"id":"https://openalex.org/I63966007","display_name":"Massachusetts Institute of Technology","ror":"https://ror.org/042nb2s44","country_code":"US","type":"education","lineage":["https://openalex.org/I63966007"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"S. Mullainathan","raw_affiliation_strings":["Massachusetts Institute of Technology and National Bureau of Economic Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Massachusetts Institute of Technology and National Bureau of Economic Research","institution_ids":["https://openalex.org/I1321305853","https://openalex.org/I63966007"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":{"value":7619,"currency":"USD","value_usd":7619},"apc_paid":null,"fwci":137.0383,"has_fulltext":false,"cited_by_count":12961,"citation_normalized_percentile":{"value":1.0,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":99,"max":100},"biblio":{"volume":"119","issue":"1","first_page":"249","last_page":"275"},"is_retracted":false,"is_pulled":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10845","display_name":"Advanced Causal Inference Techniques","score":0.9945999979972839,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10845","display_name":"Advanced Causal Inference Techniques","score":0.9945999979972839,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11911","display_name":"Spatial and Panel Data Analysis","score":0.0027000000700354576,"subfield":{"id":"https://openalex.org/subfields/2002","display_name":"Economics and Econometrics"},"field":{"id":"https://openalex.org/fields/20","display_name":"Economics, Econometrics and Finance"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10007","display_name":"Monetary Policy and Economic Impact","score":9.999999747378752e-05,"subfield":{"id":"https://openalex.org/subfields/2000","display_name":"General Economics, Econometrics and Finance"},"field":{"id":"https://openalex.org/fields/20","display_name":"Economics, Econometrics and Finance"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/difference-in-differences","display_name":"difference-in-differences","score":0.8889999985694885},{"id":"https://openalex.org/keywords/standard-errors","display_name":"standard errors","score":0.5669999718666077},{"id":"https://openalex.org/keywords/serial-correlation","display_name":"serial correlation","score":0.7440000176429749},{"id":"https://openalex.org/keywords/placebo-tests","display_name":"placebo tests","score":0.8450000286102295},{"id":"https://openalex.org/keywords/bootstrap-method","display_name":"bootstrapping","score":0.48899999260902405},{"id":"https://openalex.org/keywords/covariance-matrix","display_name":"covariance matrix","score":0.7070000171661377},{"id":"https://openalex.org/keywords/asymptotic-variance","display_name":"asymptotic variance","score":0.35899999737739563},{"id":"https://openalex.org/keywords/effective-sample-size","display_name":"effective sample size","score":0.5960000157356262},{"id":"https://openalex.org/keywords/women-s-wages","display_name":"women's wages","score":0.6320000290870667},{"id":"https://openalex.org/keywords/state-level-policy-effects","display_name":"state-level policy effects","score":0.4339999854564667},{"id":"https://openalex.org/keywords/monte-carlo-simulation","display_name":"Monte Carlo simulation","score":0.5370000004768372}],"concepts":[{"id":"https://openalex.org/C185429906","wikidata":"https://www.wikidata.org/wiki/Q1130160","display_name":"Estimator","level":2,"score":0.7294514775276184},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.6629002690315247},{"id":"https://openalex.org/C5297727","wikidata":"https://www.wikidata.org/wiki/Q786970","display_name":"Autocorrelation","level":2,"score":0.6123117208480835},{"id":"https://openalex.org/C18747219","wikidata":"https://www.wikidata.org/wiki/Q620994","display_name":"Standard error","level":2,"score":0.5879876613616943},{"id":"https://openalex.org/C143724316","wikidata":"https://www.wikidata.org/wiki/Q312468","display_name":"Series (stratigraphy)","level":2,"score":0.5699870586395264},{"id":"https://openalex.org/C22679943","wikidata":"https://www.wikidata.org/wiki/Q159375","display_name":"Standard deviation","level":2,"score":0.5667741298675537},{"id":"https://openalex.org/C196083921","wikidata":"https://www.wikidata.org/wiki/Q7915758","display_name":"Variance (accounting)","level":2,"score":0.518890917301178},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.5041519403457642},{"id":"https://openalex.org/C19499675","wikidata":"https://www.wikidata.org/wiki/Q232207","display_name":"Monte Carlo method","level":2,"score":0.49490779638290405},{"id":"https://openalex.org/C117251300","wikidata":"https://www.wikidata.org/wiki/Q1849855","display_name":"Parametric statistics","level":2,"score":0.48345598578453064},{"id":"https://openalex.org/C129848803","wikidata":"https://www.wikidata.org/wiki/Q2564360","display_name":"Sample size determination","level":2,"score":0.4716557562351227},{"id":"https://openalex.org/C2908647359","wikidata":"https://www.wikidata.org/wiki/Q2625603","display_name":"Population","level":2,"score":0.4577014446258545},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.44702035188674927},{"id":"https://openalex.org/C178650346","wikidata":"https://www.wikidata.org/wiki/Q201984","display_name":"Covariance","level":2,"score":0.41095274686813354},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.28813424706459045},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.0},{"id":"https://openalex.org/C121955636","wikidata":"https://www.wikidata.org/wiki/Q4116214","display_name":"Accounting","level":1,"score":0.0},{"id":"https://openalex.org/C149923435","wikidata":"https://www.wikidata.org/wiki/Q37732","display_name":"Demography","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0}],"mesh":[],"locations_count":10,"locations":[{"id":"doi:10.1162/003355304772839588","doi":"https://doi.org/10.1162/003355304772839588","is_oa":false,"landing_page_url":"https://doi.org/10.1162/003355304772839588","pdf_url":null,"source":{"id":"https://openalex.org/S203860005","display_name":"The Quarterly Journal of Economics","issn_l":"0033-5533","issn":["0033-5533","1531-4650"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["abdc-a-star","anvur-class-a","anvur-scientific","cwts-core","ft50","jufo-3","norway-2","poland-200","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310311648","host_organization_name":"Oxford University Press","host_organization_lineage":["https://openalex.org/P4310311648","https://openalex.org/P4310311647"],"host_organization_lineage_names":["Oxford University Press","University of Oxford"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"The Quarterly Journal of Economics","raw_type":"journal-article"},{"id":"pmh:oai:RePEc:nbr:nberwo:8841","doi":null,"is_oa":false,"landing_page_url":"http://www.nber.org/papers/w8841.pdf","pdf_url":"http://www.nber.org/papers/w8841.pdf","source":{"id":"https://openalex.org/S4306401271","display_name":"RePEc: Research Papers in Economics","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I77793887","host_organization_name":"Federal Reserve Bank of St. Louis","host_organization_lineage":["https://openalex.org/I77793887"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"preprint"},{"id":"pmh:oai:aleph.bib-bvb.de:BVB01-016904183","doi":null,"is_oa":false,"landing_page_url":"http://papers.nber.org/papers/w8841.pdf","pdf_url":"http://papers.nber.org/papers/w8841.pdf","source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:RePEc:oup:qjecon:v:119:y:2004:i:1:p:249-275.","doi":null,"is_oa":false,"landing_page_url":"http://hdl.handle.net/10.1162/003355304772839588","pdf_url":null,"source":{"id":"https://openalex.org/S4306401271","display_name":"RePEc: Research Papers in Economics","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I77793887","host_organization_name":"Federal Reserve Bank of St. Louis","host_organization_lineage":["https://openalex.org/I77793887"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article"},{"id":"pmh:oai:dspace.mit.edu:1721.1/63690","doi":null,"is_oa":true,"landing_page_url":"http://hdl.handle.net/1721.1/63690","pdf_url":null,"source":{"id":"https://openalex.org/S4306400425","display_name":"DSpace@MIT (Massachusetts Institute of Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I63966007","host_organization_name":"Massachusetts Institute of Technology","host_organization_lineage":["https://openalex.org/I63966007"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Working Paper"},{"id":"pmh:oai:CiteSeerX.psu:10.1.1.1032.8405","doi":null,"is_oa":false,"landing_page_url":"http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.1032.8405","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://piketty.pse.ens.fr/fichiers/Bertrandetal2003.pdf","raw_type":"text"},{"id":"pmh:oai:CiteSeerX.psu:10.1.1.320.5848","doi":null,"is_oa":false,"landing_page_url":"http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.320.5848","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://faculty.arts.ubc.ca/tlemieux/econ628/BDM.pdf","raw_type":"text"},{"id":"pmh:oai:CiteSeerX.psu:10.1.1.559.6555","doi":null,"is_oa":false,"landing_page_url":"http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.559.6555","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://faculty.smu.edu/millimet/classes/eco7377/papers/bertrand et al.pdf","raw_type":"text"},{"id":"pmh:oai:CiteSeerX.psu:10.1.1.694.484","doi":null,"is_oa":false,"landing_page_url":"http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.694.484","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://www.nber.org/papers/w8841.pdf","raw_type":"text"},{"id":"pmh:oai:CiteSeerX.psu:10.1.1.721.5260","doi":null,"is_oa":false,"landing_page_url":"http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.721.5260","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://faculty.chicagobooth.edu/marianne.bertrand/research/papers/dif_in_dif_qje.pdf","raw_type":"text"}],"best_oa_location":{"id":"pmh:oai:dspace.mit.edu:1721.1/63690","doi":null,"is_oa":true,"landing_page_url":"http://hdl.handle.net/1721.1/63690","pdf_url":null,"source":{"id":"https://openalex.org/S4306400425","display_name":"DSpace@MIT (Massachusetts Institute of Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I63966007","host_organization_name":"Massachusetts Institute of Technology","host_organization_lineage":["https://openalex.org/I63966007"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Working Paper"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":25,"referenced_works":["https://openalex.org/W600065214","https://openalex.org/W1562778220","https://openalex.org/W1593250692","https://openalex.org/W1980412904","https://openalex.org/W1984935883","https://openalex.org/W1989581591","https://openalex.org/W1998653759","https://openalex.org/W2007212082","https://openalex.org/W2009729418","https://openalex.org/W2047670349","https://openalex.org/W2066860374","https://openalex.org/W2111201888","https://openalex.org/W2117178635","https://openalex.org/W2118859991","https://openalex.org/W2124758339","https://openalex.org/W2128017766","https://openalex.org/W2134246329","https://openalex.org/W2157635736","https://openalex.org/W3023906401","https://openalex.org/W3121273981","https://openalex.org/W3121353515","https://openalex.org/W3125512578","https://openalex.org/W4237954859","https://openalex.org/W4394945775","https://openalex.org/W7054757705"],"related_works":["https://openalex.org/W3194695304","https://openalex.org/W2587235528","https://openalex.org/W4410116931","https://openalex.org/W2470558841","https://openalex.org/W3121628497","https://openalex.org/W2938349878","https://openalex.org/W2476848909","https://openalex.org/W2999711672","https://openalex.org/W4410776556"],"abstract_inverted_index":{"Most":[0],"papers":[1],"that":[2,20,122,175],"employ":[3],"Differences-in-Differences":[4],"estimation":[5],"(DD)":[6],"use":[7,54,106],"many":[8],"years":[9],"of":[10,31,61,70,83,101,142,150,161,170,201],"data":[11,41],"and":[12,172,184,186],"focus":[13],"on":[14,42,128,158],"serially":[15],"correlated":[16],"outcomes":[17],"but":[18],"ignore":[19],"the":[21,29,46,58,67,80,84,92,102,129,140,143,148,162,177,191],"resulting":[22],"standard":[23,68,76,81],"errors":[24,77],"are":[25],"inconsistent.":[26],"To":[27],"illustrate":[28],"severity":[30],"this":[32,71,118],"issue,":[33],"we":[34,53,86],"randomly":[35],"generate":[36],"placebo":[37,103],"laws":[38],"in":[39],"state-level":[40],"female":[43],"wages":[44],"from":[45],"Current":[47],"Population":[48],"Survey.":[49],"For":[50],"each":[51],"law,":[52],"OLS":[55],"to":[56,98,110],"compute":[57],"DD":[59,75],"estimate":[60],"its":[62],"\u201ceffect\u201d":[63,89],"as":[64,66],"well":[65,113,146,166,196],"error":[69],"estimate.":[72],"These":[73],"conventional":[74],"severely":[78],"understate":[79],"deviation":[82],"estimators:":[85],"find":[87],"an":[88],"significant":[90],"at":[91],"5":[93],"percent":[94,100],"level":[95],"for":[96,167,198],"up":[97],"45":[99],"interventions.":[104],"We":[105],"Monte":[107],"Carlo":[108],"simulations":[109],"investigate":[111],"how":[112],"existing":[114],"methods":[115],"help":[116],"solve":[117],"problem.":[119],"Econometric":[120],"corrections":[121,156],"place":[123],"a":[124,182],"specific":[125],"parametric":[126],"form":[127],"time-series":[130],"process":[131],"do":[132],"not":[133],"perform":[134],"well.":[135],"Bootstrap":[136],"(taking":[137],"into":[138,181,189],"account":[139,190],"autocorrelation":[141],"data)":[144],"works":[145,195],"when":[147],"number":[149],"states":[151,171],"is":[152],"large":[153],"enough.":[154],"Two":[155],"based":[157],"asymptotic":[159],"approximation":[160],"variance-covariance":[163],"matrix":[164],"work":[165],"moderate":[168],"numbers":[169,200],"one":[173],"correction":[174],"collapses":[176],"time":[178],"series":[179],"information":[180],"\u201cpre\u201d-":[183],"\u201cpost\u201d-period":[185],"explicitly":[187],"takes":[188],"effective":[192],"sample":[193],"size":[194],"even":[197],"small":[199],"states.":[202]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W2096684396","counts_by_year":[{"year":2026,"cited_by_count":1187},{"year":2025,"cited_by_count":856},{"year":2024,"cited_by_count":764},{"year":2023,"cited_by_count":747},{"year":2022,"cited_by_count":739},{"year":2021,"cited_by_count":848},{"year":2020,"cited_by_count":876},{"year":2019,"cited_by_count":742},{"year":2018,"cited_by_count":735},{"year":2017,"cited_by_count":760},{"year":2016,"cited_by_count":671},{"year":2015,"cited_by_count":592},{"year":2014,"cited_by_count":591},{"year":2013,"cited_by_count":519},{"year":2012,"cited_by_count":489}],"updated_date":"2026-11-19T14:11:59.513116","created_date":"2016-06-24T00:00:00"}
