{"id":"https://openalex.org/W3124589439","doi":"https://doi.org/10.1257/mic.20200129","title":"Stability of Experimental Results: Forecasts and Evidence","display_name":"Stability of Experimental Results: Forecasts and Evidence","publication_year":2019,"publication_date":"2019-01-01","ids":{"openalex":"https://openalex.org/W3124589439","doi":"https://doi.org/10.1257/mic.20200129","mag":"3124589439"},"language":"en","primary_location":{"id":"doi:10.1257/mic.20200129","doi":"https://doi.org/10.1257/mic.20200129","is_oa":false,"landing_page_url":"https://doi.org/10.1257/mic.20200129","pdf_url":null,"source":{"id":"https://openalex.org/S96919139","display_name":"American Economic Journal Microeconomics","issn_l":"1945-7669","issn":["1945-7669","1945-7685"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["abdc-a-star","anvur-class-a","anvur-scientific","cwts-core","jufo-3","ki-jl-1","norway-2","poland-200","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310315912","host_organization_name":"American Economic Association","host_organization_lineage":["https://openalex.org/P4310315912"],"host_organization_lineage_names":["American Economic Association"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"American Economic Journal: Microeconomics","raw_type":"journal-article"},"volume":null,"type":"article","is_traditional":true,"indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.2139/ssrn.3391520","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5111929267","display_name":"Stefano DellaVigna","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I1321305853","display_name":"National Bureau of Economic Research","ror":"https://ror.org/04grmx538","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I1321305853"]},{"id":"https://openalex.org/I95457486","display_name":"University of California, Berkeley","ror":"https://ror.org/01an7q238","country_code":"US","type":"education","lineage":["https://openalex.org/I95457486"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Stefano DellaVigna","raw_affiliation_strings":["Department of Economics, UC Berkeley and NBER (email: )"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Economics, UC Berkeley and NBER (email: )","institution_ids":["https://openalex.org/I1321305853","https://openalex.org/I95457486"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5085766707","display_name":"Devin G. Pope","orcid":"https://orcid.org/0000-0002-6567-9654","observed_orcids":["https://orcid.org/0000-0002-6567-9654"]},"institutions":[{"id":"https://openalex.org/I40347166","display_name":"University of Chicago","ror":"https://ror.org/024mw5h28","country_code":"US","type":"education","lineage":["https://openalex.org/I40347166"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Devin Pope","raw_affiliation_strings":["Booth School of Business, University of Chicago and NBER (email: )"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Booth School of Business, University of Chicago and NBER (email: )","institution_ids":["https://openalex.org/I40347166"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":2.371,"has_fulltext":true,"cited_by_count":19,"citation_normalized_percentile":{"value":0.89847218,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":87,"max":98},"biblio":{"volume":"14","issue":"3","first_page":"889","last_page":"925"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10646","display_name":"Experimental Behavioral Economics Studies","score":0.9961000084877014,"subfield":{"id":"https://openalex.org/subfields/2002","display_name":"Economics and Econometrics"},"field":{"id":"https://openalex.org/fields/20","display_name":"Economics, Econometrics and Finance"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10646","display_name":"Experimental Behavioral Economics Studies","score":0.9961000084877014,"subfield":{"id":"https://openalex.org/subfields/2002","display_name":"Economics and Econometrics"},"field":{"id":"https://openalex.org/fields/20","display_name":"Economics, Econometrics and Finance"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10315","display_name":"Decision-Making and Behavioral Economics","score":0.00139999995008111,"subfield":{"id":"https://openalex.org/subfields/1800","display_name":"General Decision Sciences"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10991","display_name":"Game Theory and Voting Systems","score":0.00039999998989515007,"subfield":{"id":"https://openalex.org/subfields/2002","display_name":"Economics and Econometrics"},"field":{"id":"https://openalex.org/fields/20","display_name":"Economics, Econometrics and Finance"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/replication","display_name":"replication","score":0.7329999804496765},{"id":"https://openalex.org/keywords/real-effort-task","display_name":"real-effort task","score":0.6039999723434448},{"id":"https://openalex.org/keywords/behavioral-therapy","display_name":"behavior therapy","score":0.6330000162124634},{"id":"https://openalex.org/keywords/population-stability","display_name":"population stability","score":0.5410000085830688},{"id":"https://openalex.org/keywords/geography-and-culture","display_name":"geography and culture","score":0.6869999766349792},{"id":"https://openalex.org/keywords/task-change","display_name":"task change","score":0.5239999890327454},{"id":"https://openalex.org/keywords/output-measures","display_name":"output measures","score":0.6880000233650208},{"id":"https://openalex.org/keywords/informed-consent-forms","display_name":"informed consent forms","score":0.6650000214576721},{"id":"https://openalex.org/keywords/expert-expectations","display_name":"expert expectations","score":0.3619999885559082},{"id":"https://openalex.org/keywords/experimental-design","display_name":"experimental design","score":0.3580000102519989}],"concepts":[{"id":"https://openalex.org/C12590798","wikidata":"https://www.wikidata.org/wiki/Q3933199","display_name":"Replication (statistics)","level":2,"score":0.8122876882553101},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.7282992005348206},{"id":"https://openalex.org/C112972136","wikidata":"https://www.wikidata.org/wiki/Q7595718","display_name":"Stability (learning theory)","level":2,"score":0.7229747176170349},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.7034103870391846},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.6724077463150024},{"id":"https://openalex.org/C2780084366","wikidata":"https://www.wikidata.org/wiki/Q37732","display_name":"Demographics","level":2,"score":0.6467337012290955},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.5195841789245605},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.5193494558334351},{"id":"https://openalex.org/C99498987","wikidata":"https://www.wikidata.org/wiki/Q2210247","display_name":"Noise (video)","level":3,"score":0.49671846628189087},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.4783308506011963},{"id":"https://openalex.org/C182306322","wikidata":"https://www.wikidata.org/wiki/Q1779371","display_name":"Order (exchange)","level":2,"score":0.4474048912525177},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.44224339723587036},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.39890119433403015},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.3553324043750763},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.34966397285461426},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3357533812522888},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.3319319486618042},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.23472583293914795},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.21019214391708374},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.18907573819160461},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.18133977055549622},{"id":"https://openalex.org/C149923435","wikidata":"https://www.wikidata.org/wiki/Q37732","display_name":"Demography","level":1,"score":0.16310235857963562},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.10021266341209412},{"id":"https://openalex.org/C10138342","wikidata":"https://www.wikidata.org/wiki/Q43015","display_name":"Finance","level":1,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C166957645","wikidata":"https://www.wikidata.org/wiki/Q23498","display_name":"Archaeology","level":1,"score":0.0},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.0},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0}],"mesh":[],"locations_count":5,"locations":[{"id":"doi:10.1257/mic.20200129","doi":"https://doi.org/10.1257/mic.20200129","is_oa":false,"landing_page_url":"https://doi.org/10.1257/mic.20200129","pdf_url":null,"source":{"id":"https://openalex.org/S96919139","display_name":"American Economic Journal Microeconomics","issn_l":"1945-7669","issn":["1945-7669","1945-7685"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["abdc-a-star","anvur-class-a","anvur-scientific","cwts-core","jufo-3","ki-jl-1","norway-2","poland-200","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310315912","host_organization_name":"American Economic Association","host_organization_lineage":["https://openalex.org/P4310315912"],"host_organization_lineage_names":["American Economic Association"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"American Economic Journal: Microeconomics","raw_type":"journal-article"},{"id":"doi:10.2139/ssrn.3391520","doi":"https://doi.org/10.2139/ssrn.3391520","is_oa":true,"landing_page_url":"https://doi.org/10.2139/ssrn.3391520","pdf_url":null,"source":{"id":"https://openalex.org/S4210172589","display_name":"SSRN Electronic Journal","issn_l":"1556-5068","issn":["1556-5068"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I1318003438","host_organization_name":"RELX Group (Netherlands)","host_organization_lineage":["https://openalex.org/I1318003438"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"SSRN Electronic Journal","raw_type":"journal-article"},{"id":"pmh:oai:RePEc:nbr:nberwo:25858","doi":null,"is_oa":true,"landing_page_url":"http://www.nber.org/papers/w25858.pdf","pdf_url":"http://www.nber.org/papers/w25858.pdf","source":{"id":"https://openalex.org/S4306401271","display_name":"RePEc: Research Papers in Economics","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I77793887","host_organization_name":"Federal Reserve Bank of St. Louis","host_organization_lineage":["https://openalex.org/I77793887"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"preprint"},{"id":"mag:3124589439","doi":null,"is_oa":false,"landing_page_url":"https://econpapers.repec.org/paper/nbrnberwo/25858.htm","pdf_url":null,"source":{"id":"https://openalex.org/S4306401271","display_name":"RePEc: Research Papers in Economics","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I77793887","host_organization_name":"Federal Reserve Bank of St. Louis","host_organization_lineage":["https://openalex.org/I77793887"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"RePEc: Research Papers in Economics","raw_type":null},{"id":"pmh:oai:RePEc:aea:aejmic:v:14:y:2022:i:3:p:889-925","doi":null,"is_oa":false,"landing_page_url":"https://www.aeaweb.org/doi/10.1257/mic.20200129","pdf_url":null,"source":{"id":"https://openalex.org/S4306401271","display_name":"RePEc: Research Papers in Economics","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I77793887","host_organization_name":"Federal Reserve Bank of St. Louis","host_organization_lineage":["https://openalex.org/I77793887"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"doi:10.2139/ssrn.3391520","doi":"https://doi.org/10.2139/ssrn.3391520","is_oa":true,"landing_page_url":"https://doi.org/10.2139/ssrn.3391520","pdf_url":null,"source":{"id":"https://openalex.org/S4210172589","display_name":"SSRN Electronic Journal","issn_l":"1556-5068","issn":["1556-5068"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I1318003438","host_organization_name":"RELX Group (Netherlands)","host_organization_lineage":["https://openalex.org/I1318003438"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"SSRN Electronic Journal","raw_type":"journal-article"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3124589439.pdf","grobid_xml":"https://content.openalex.org/works/W3124589439.grobid-xml"},"referenced_works_count":38,"referenced_works":["https://openalex.org/W1853301790","https://openalex.org/W1897139626","https://openalex.org/W1941783251","https://openalex.org/W1976624377","https://openalex.org/W1997809363","https://openalex.org/W2015712245","https://openalex.org/W2026117454","https://openalex.org/W2088212458","https://openalex.org/W2099161251","https://openalex.org/W2106605967","https://openalex.org/W2143326716","https://openalex.org/W2144383963","https://openalex.org/W2145409614","https://openalex.org/W2159839307","https://openalex.org/W2165949980","https://openalex.org/W2169411968","https://openalex.org/W2275651244","https://openalex.org/W2292391400","https://openalex.org/W2346258676","https://openalex.org/W2356474243","https://openalex.org/W2561942675","https://openalex.org/W2568747767","https://openalex.org/W2584093829","https://openalex.org/W2622015926","https://openalex.org/W2625865465","https://openalex.org/W2776961836","https://openalex.org/W2884233016","https://openalex.org/W2886512263","https://openalex.org/W2887346961","https://openalex.org/W2892216368","https://openalex.org/W2991191531","https://openalex.org/W3110959118","https://openalex.org/W3122012753","https://openalex.org/W3123554691","https://openalex.org/W3124211044","https://openalex.org/W3124878131","https://openalex.org/W3124900509","https://openalex.org/W4251446432"],"related_works":["https://openalex.org/W2330345543","https://openalex.org/W4323075630","https://openalex.org/W7212655747","https://openalex.org/W1519845978","https://openalex.org/W1982110736","https://openalex.org/W2305796456","https://openalex.org/W7098959923","https://openalex.org/W4312329205","https://openalex.org/W2033676480","https://openalex.org/W7070884800"],"abstract_inverted_index":{"How":[0],"robust":[1],"are":[2,153,173,197],"experimental":[3,135],"results":[4,107,142,152,172,227],"to":[5,75,89,103,118,168,176],"changes":[6,14,117],"in":[7],"design?":[8],"And":[9],"can":[10],"researchers":[11],"anticipate":[12],"which":[13],"matter":[15],"most?":[16],"We":[17,59,128,217],"consider":[18],"a":[19,22,56,79,125],"specific":[20],"context,":[21],"real-effort":[23],"task":[24,120,157,164],"with":[25],"multiple":[26],"behavioral":[27],"treatments,":[28],"and":[29,43,70,88,115,121,137,158,225],"examine":[30],"the":[31,46,49,53,64,72,76,97,106,119,134,141,148,163,169,177,182,191,219,223,226],"stability":[32,83,139,184],"along":[33],"six":[34],"dimensions:":[35],"(i)":[36],"pure":[37,98],"replication;":[38],"(ii)":[39],"demographics;":[40],"(iii)":[41],"geography":[42],"culture;":[44],"(iv)":[45],"task;":[47],"(v)":[48],"output":[50,122,159],"measure;":[51],"(vi)":[52],"presence":[54],"of":[55,68,81,133,140,179,190,221],"consent":[57],"form.":[58],"use":[60],"rank-order":[61],"correlation":[62,74],"across":[63,111,143,156],"treatments":[65],"as":[66,212,214],"measure":[67],"stability,":[69,193],"compare":[71],"observed":[73,192],"one":[77],"under":[78],"benchmark":[80,185],"full":[82,138,183],"(which":[84],"allows":[85],"for":[86,228],"noise),":[87],"expert":[90,195],"forecasts.":[91],"The":[92,151,171],"academic":[93],"experts":[94,149],"expect":[95],"that":[96,105,116,199,203],"replication":[99,132],"will":[100,108,123],"be":[101,211],"close":[102],"perfect,":[104],"differ":[109],"sizably":[110],"demographic":[112],"groups":[113],"(age/gender/education),":[114],"make":[124],"further":[126],"impact.":[127],"find":[129],"near":[130],"perfect":[131],"results,":[136],"demographics,":[144],"significantly":[145],"higher":[146],"than":[147],"expected.":[150],"quite":[154],"different":[155],"change,":[160],"mostly":[161],"because":[162],"change":[165],"adds":[166],"noise":[167],"findings.":[170],"also":[174],"stable":[175],"lack":[178],"consent.":[180],"Overall,":[181],"is":[186],"an":[187],"excellent":[188],"predictor":[189],"while":[194],"forecasts":[196],"not":[198,210],"informative.":[200],"This":[201],"suggests":[202],"researchers'":[204],"predictions":[205],"about":[206],"external":[207],"validity":[208],"may":[209],"informative":[213],"they":[215],"expect.":[216],"discuss":[218],"implications":[220],"both":[222],"methods":[224],"conceptual":[229],"replication.":[230]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":5},{"year":2023,"cited_by_count":3},{"year":2022,"cited_by_count":4},{"year":2021,"cited_by_count":1},{"year":2020,"cited_by_count":2},{"year":2019,"cited_by_count":1}],"updated_date":"2026-11-10T11:39:43.021295","created_date":"2021-02-01T00:00:00"}