{"id":"https://openalex.org/W3196934205","doi":"https://doi.org/10.1145/3468267.3470572","title":"Ensuring statistical reproducibility of ocean model simulations in the age of hybrid computing","display_name":"Ensuring statistical reproducibility of ocean model simulations in the age of hybrid computing","publication_year":2021,"publication_date":"2021-07-05","ids":{"openalex":"https://openalex.org/W3196934205","doi":"https://doi.org/10.1145/3468267.3470572","mag":"3196934205"},"language":"en","primary_location":{"id":"doi:10.1145/3468267.3470572","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3468267.3470572","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Platform for Advanced Scientific Computing Conference","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://www.osti.gov/servlets/purl/1820842","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5023485909","display_name":"Salil Mahajan","orcid":"https://orcid.org/0000-0001-5767-8590"},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Salil Mahajan","raw_affiliation_strings":["Computational Earth Sciences"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Computational Earth Sciences","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":["https://openalex.org/A5023485909"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":4.8502,"has_fulltext":true,"cited_by_count":9,"citation_normalized_percentile":{"value":0.94987985,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"9"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10466","display_name":"Meteorological Phenomena and Simulations","score":0.9887999892234802,"subfield":{"id":"https://openalex.org/subfields/1902","display_name":"Atmospheric Science"},"field":{"id":"https://openalex.org/fields/19","display_name":"Earth and Planetary Sciences"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10466","display_name":"Meteorological Phenomena and Simulations","score":0.9887999892234802,"subfield":{"id":"https://openalex.org/subfields/1902","display_name":"Atmospheric Science"},"field":{"id":"https://openalex.org/fields/19","display_name":"Earth and Planetary Sciences"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10255","display_name":"Oceanographic and Atmospheric Processes","score":0.9799000024795532,"subfield":{"id":"https://openalex.org/subfields/1910","display_name":"Oceanography"},"field":{"id":"https://openalex.org/fields/19","display_name":"Earth and Planetary Sciences"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11801","display_name":"Reservoir Engineering and Simulation Methods","score":0.9775000214576721,"subfield":{"id":"https://openalex.org/subfields/2212","display_name":"Ocean Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7095758318901062},{"id":"https://openalex.org/keywords/ensemble-forecasting","display_name":"Ensemble forecasting","score":0.5582975745201111},{"id":"https://openalex.org/keywords/suite","display_name":"Suite","score":0.5400879979133606},{"id":"https://openalex.org/keywords/climate-model","display_name":"Climate model","score":0.5149105787277222},{"id":"https://openalex.org/keywords/null","display_name":"Null (SQL)","score":0.5145953893661499},{"id":"https://openalex.org/keywords/statistical-power","display_name":"Statistical power","score":0.4863879978656769},{"id":"https://openalex.org/keywords/monte-carlo-method","display_name":"Monte Carlo method","score":0.475475937128067},{"id":"https://openalex.org/keywords/statistical-hypothesis-testing","display_name":"Statistical hypothesis testing","score":0.47286319732666016},{"id":"https://openalex.org/keywords/test-suite","display_name":"Test suite","score":0.45789745450019836},{"id":"https://openalex.org/keywords/code-refactoring","display_name":"Code refactoring","score":0.42493486404418945},{"id":"https://openalex.org/keywords/energy","display_name":"Energy (signal processing)","score":0.4158400297164917},{"id":"https://openalex.org/keywords/sample-size-determination","display_name":"Sample size determination","score":0.41529133915901184},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.3584613800048828},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.30926936864852905},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.3087875545024872},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.22407320141792297},{"id":"https://openalex.org/keywords/test-case","display_name":"Test case","score":0.1752430498600006},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.15856105089187622},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.15658462047576904},{"id":"https://openalex.org/keywords/climate-change","display_name":"Climate change","score":0.14507809281349182},{"id":"https://openalex.org/keywords/regression-analysis","display_name":"Regression analysis","score":0.12806841731071472}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7095758318901062},{"id":"https://openalex.org/C119898033","wikidata":"https://www.wikidata.org/wiki/Q3433888","display_name":"Ensemble forecasting","level":2,"score":0.5582975745201111},{"id":"https://openalex.org/C79581498","wikidata":"https://www.wikidata.org/wiki/Q1367530","display_name":"Suite","level":2,"score":0.5400879979133606},{"id":"https://openalex.org/C168754636","wikidata":"https://www.wikidata.org/wiki/Q620920","display_name":"Climate model","level":3,"score":0.5149105787277222},{"id":"https://openalex.org/C203763787","wikidata":"https://www.wikidata.org/wiki/Q371029","display_name":"Null (SQL)","level":2,"score":0.5145953893661499},{"id":"https://openalex.org/C96608239","wikidata":"https://www.wikidata.org/wiki/Q1199823","display_name":"Statistical power","level":2,"score":0.4863879978656769},{"id":"https://openalex.org/C19499675","wikidata":"https://www.wikidata.org/wiki/Q232207","display_name":"Monte Carlo method","level":2,"score":0.475475937128067},{"id":"https://openalex.org/C87007009","wikidata":"https://www.wikidata.org/wiki/Q210832","display_name":"Statistical hypothesis testing","level":2,"score":0.47286319732666016},{"id":"https://openalex.org/C151552104","wikidata":"https://www.wikidata.org/wiki/Q7705809","display_name":"Test suite","level":4,"score":0.45789745450019836},{"id":"https://openalex.org/C152752567","wikidata":"https://www.wikidata.org/wiki/Q116877","display_name":"Code refactoring","level":3,"score":0.42493486404418945},{"id":"https://openalex.org/C186370098","wikidata":"https://www.wikidata.org/wiki/Q442787","display_name":"Energy (signal processing)","level":2,"score":0.4158400297164917},{"id":"https://openalex.org/C129848803","wikidata":"https://www.wikidata.org/wiki/Q2564360","display_name":"Sample size determination","level":2,"score":0.41529133915901184},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.3584613800048828},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.30926936864852905},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3087875545024872},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.22407320141792297},{"id":"https://openalex.org/C128942645","wikidata":"https://www.wikidata.org/wiki/Q1568346","display_name":"Test case","level":3,"score":0.1752430498600006},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.15856105089187622},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.15658462047576904},{"id":"https://openalex.org/C132651083","wikidata":"https://www.wikidata.org/wiki/Q7942","display_name":"Climate change","level":2,"score":0.14507809281349182},{"id":"https://openalex.org/C152877465","wikidata":"https://www.wikidata.org/wiki/Q208042","display_name":"Regression analysis","level":2,"score":0.12806841731071472},{"id":"https://openalex.org/C18903297","wikidata":"https://www.wikidata.org/wiki/Q7150","display_name":"Ecology","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C166957645","wikidata":"https://www.wikidata.org/wiki/Q23498","display_name":"Archaeology","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C95457728","wikidata":"https://www.wikidata.org/wiki/Q309","display_name":"History","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3468267.3470572","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3468267.3470572","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Platform for Advanced Scientific Computing Conference","raw_type":"proceedings-article"},{"id":"pmh:oai:osti.gov:1820842","is_oa":true,"landing_page_url":"https://www.osti.gov/biblio/1820842","pdf_url":"https://www.osti.gov/servlets/purl/1820842","source":{"id":"https://openalex.org/S4306402487","display_name":"OSTI OAI (U.S. Department of Energy Office of Scientific and Technical Information)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I139351228","host_organization_name":"Office of Scientific and Technical Information","host_organization_lineage":["https://openalex.org/I139351228"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":null}],"best_oa_location":{"id":"pmh:oai:osti.gov:1820842","is_oa":true,"landing_page_url":"https://www.osti.gov/biblio/1820842","pdf_url":"https://www.osti.gov/servlets/purl/1820842","source":{"id":"https://openalex.org/S4306402487","display_name":"OSTI OAI (U.S. Department of Energy Office of Scientific and Technical Information)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I139351228","host_organization_name":"Office of Scientific and Technical Information","host_organization_lineage":["https://openalex.org/I139351228"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":null},"sustainable_development_goals":[{"score":0.46000000834465027,"display_name":"Climate action","id":"https://metadata.un.org/sdg/13"},{"score":0.4099999964237213,"display_name":"Life below water","id":"https://metadata.un.org/sdg/14"}],"awards":[{"id":"https://openalex.org/G102163711","display_name":null,"funder_award_id":"DE\u2010 AC02\u201006CH11357","funder_id":"https://openalex.org/F4320306084","funder_display_name":"U.S. Department of Energy"},{"id":"https://openalex.org/G107546177","display_name":null,"funder_award_id":"DE- AC02-06-CH11357","funder_id":"https://openalex.org/F4320338284","funder_display_name":"Argonne National Laboratory"},{"id":"https://openalex.org/G121915436","display_name":null,"funder_award_id":"DEAC05- 00OR22725","funder_id":"https://openalex.org/F4320332359","funder_display_name":"Office of Science"},{"id":"https://openalex.org/G1286236842","display_name":null,"funder_award_id":"-AC02-05CH11231","funder_id":"https://openalex.org/F4320306084","funder_display_name":"U.S. Department of Energy"},{"id":"https://openalex.org/G1313983767","display_name":null,"funder_award_id":"DE-AC02","funder_id":"https://openalex.org/F4320338284","funder_display_name":"Argonne National Laboratory"},{"id":"https://openalex.org/G1337544421","display_name":null,"funder_award_id":"05CH11231","funder_id":"https://openalex.org/F4320337509","funder_display_name":"Biological and Environmental Research"},{"id":"https://openalex.org/G1528409558","display_name":null,"funder_award_id":"No. DE-AC05-00OR22725","funder_id":"https://openalex.org/F4320332359","funder_display_name":"Office of Science"},{"id":"https://openalex.org/G1645119126","display_name":null,"funder_award_id":"AC05-00OR22725","funder_id":"https://openalex.org/F4320306084","funder_display_name":"U.S. Department of Energy"},{"id":"https://openalex.org/G1664757510","display_name":null,"funder_award_id":"AC02\u201306CH11357","funder_id":"https://openalex.org/F4320338284","funder_display_name":"Argonne National Laboratory"},{"id":"https://openalex.org/G203625172","display_name":null,"funder_award_id":"DE--AC05--00OR22725","funder_id":"https://openalex.org/F4320306084","funder_display_name":"U.S. Department of Energy"},{"id":"https://openalex.org/G223002938","display_name":null,"funder_award_id":"DE-AC02 \u2212 05CH11231","funder_id":"https://openalex.org/F4320337509","funder_display_name":"Biological and Environmental Research"},{"id":"https://openalex.org/G2287611682","display_name":null,"funder_award_id":"AC02\u201006CH11357","funder_id":"https://openalex.org/F4320332359","funder_display_name":"Office of Science"},{"id":"https://openalex.org/G262962599","display_name":null,"funder_award_id":"AC02\u201006CH11357","funder_id":"https://openalex.org/F4320306084","funder_display_name":"U.S. Department of Energy"},{"id":"https://openalex.org/G2638771445","display_name":null,"funder_award_id":"DE-AC02-05CH11231","funder_id":"https://openalex.org/F4320338284","funder_display_name":"Argonne National Laboratory"},{"id":"https://openalex.org/G2857925935","display_name":null,"funder_award_id":"DEAC0206CH11357","funder_id":"https://openalex.org/F4320337509","funder_display_name":"Biological and Environmental Research"},{"id":"https://openalex.org/G2988778103","display_name":null,"funder_award_id":"DE-AC05?","funder_id":"https://openalex.org/F4320306084","funder_display_name":"U.S. Department of Energy"},{"id":"https://openalex.org/G2998698001","display_name":null,"funder_award_id":"DE-AC02-06CH11357","funder_id":"https://openalex.org/F4320306250","funder_display_name":"Battelle"},{"id":"https://openalex.org/G3049477530","display_name":null,"funder_award_id":"Contract No. DE-AC05-00OR22725","funder_id":"https://openalex.org/F4320306084","funder_display_name":"U.S. Department of Energy"},{"id":"https://openalex.org/G3075337988","display_name":null,"funder_award_id":"06CH11357","funder_id":"https://openalex.org/F4320338284","funder_display_name":"Argonne National Laboratory"},{"id":"https://openalex.org/G3083819904","display_name":null,"funder_award_id":"05CH11231","funder_id":"https://openalex.org/F4320306084","funder_display_name":"U.S. Department of Energy"},{"id":"https://openalex.org/G3121793570","display_name":null,"funder_award_id":"00OR22725","funder_id":"https://openalex.org/F4320337509","funder_display_name":"Biological and Environmental Research"},{"id":"https://openalex.org/G313676650","display_name":null,"funder_award_id":"DEAC05-00OR22725","funder_id":"https://openalex.org/F4320316892","funder_display_name":"UT-Battelle"},{"id":"https://openalex.org/G419881185","display_name":null,"funder_award_id":"DEAC02\u201305CH11231","funder_id":"https://openalex.org/F4320332359","funder_display_name":"Office of Science"},{"id":"https://openalex.org/G42916606","display_name":null,"funder_award_id":"DE AC02 05CH11231","funder_id":"https://openalex.org/F4320306084","funder_display_name":"U.S. Department of Energy"},{"id":"https://openalex.org/G4501827968","display_name":null,"funder_award_id":"AC02-05CH11231","funder_id":"https://openalex.org/F4320332359","funder_display_name":"Office of Science"},{"id":"https://openalex.org/G450780475","display_name":null,"funder_award_id":"DE\u2010AC05\u201300OR22725","funder_id":"https://openalex.org/F4320337509","funder_display_name":"Biological and Environmental Research"},{"id":"https://openalex.org/G498139845","display_name":null,"funder_award_id":"DE-AC02","funder_id":"https://openalex.org/F4320332359","funder_display_name":"Office of Science"},{"id":"https://openalex.org/G5726405315","display_name":null,"funder_award_id":"DE-AC05","funder_id":"https://openalex.org/F4320306250","funder_display_name":"Battelle"},{"id":"https://openalex.org/G6129992089","display_name":null,"funder_award_id":"DE-AC05-","funder_id":"https://openalex.org/F4320316892","funder_display_name":"UT-Battelle"},{"id":"https://openalex.org/G6848031779","display_name":null,"funder_award_id":"06CH11357","funder_id":"https://openalex.org/F4320306084","funder_display_name":"U.S. Department of Energy"},{"id":"https://openalex.org/G6911681599","display_name":null,"funder_award_id":"No. DE-AC05-00OR22725","funder_id":"https://openalex.org/F4320306084","funder_display_name":"U.S. Department of Energy"},{"id":"https://openalex.org/G6918803902","display_name":null,"funder_award_id":"06CH11357","funder_id":"https://openalex.org/F4320332359","funder_display_name":"Office of Science"},{"id":"https://openalex.org/G7769062580","display_name":null,"funder_award_id":"No. DE-AC05-00OR22725","funder_id":"https://openalex.org/F4320316892","funder_display_name":"UT-Battelle"},{"id":"https://openalex.org/G7787287669","display_name":null,"funder_award_id":"AC02-05CH11231","funder_id":"https://openalex.org/F4320337509","funder_display_name":"Biological and Environmental Research"},{"id":"https://openalex.org/G8269158468","display_name":null,"funder_award_id":"AC05-00OR22725","funder_id":"https://openalex.org/F4320332359","funder_display_name":"Office of Science"},{"id":"https://openalex.org/G8906985441","display_name":null,"funder_award_id":"00OR22725","funder_id":"https://openalex.org/F4320306084","funder_display_name":"U.S. Department of Energy"},{"id":"https://openalex.org/G8943143067","display_name":null,"funder_award_id":"AC05-00OR22725","funder_id":"https://openalex.org/F4320316892","funder_display_name":"UT-Battelle"},{"id":"https://openalex.org/G909711816","display_name":null,"funder_award_id":"DE-AC0500OR22725","funder_id":"https://openalex.org/F4320306250","funder_display_name":"Battelle"},{"id":"https://openalex.org/G969889393","display_name":null,"funder_award_id":"DE-AC02-","funder_id":"https://openalex.org/F4320306084","funder_display_name":"U.S. Department of Energy"},{"id":"https://openalex.org/G993149817","display_name":null,"funder_award_id":"DE-AC02-06CH11357 ","funder_id":"https://openalex.org/F4320332359","funder_display_name":"Office of Science"}],"funders":[{"id":"https://openalex.org/F4320306084","display_name":"U.S. Department of Energy","ror":"https://ror.org/01bj3aw27"},{"id":"https://openalex.org/F4320306250","display_name":"Battelle","ror":"https://ror.org/01h5tnr73"},{"id":"https://openalex.org/F4320316892","display_name":"UT-Battelle","ror":"https://ror.org/04nza6677"},{"id":"https://openalex.org/F4320332359","display_name":"Office of Science","ror":"https://ror.org/00mmn6b08"},{"id":"https://openalex.org/F4320337509","display_name":"Biological and Environmental Research","ror":"https://ror.org/0114b2m14"},{"id":"https://openalex.org/F4320338284","display_name":"Argonne National Laboratory","ror":"https://ror.org/05gvnxz63"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3196934205.pdf","grobid_xml":"https://content.openalex.org/works/W3196934205.grobid-xml"},"referenced_works_count":17,"referenced_works":["https://openalex.org/W1616657595","https://openalex.org/W1940560916","https://openalex.org/W1990548731","https://openalex.org/W2020699102","https://openalex.org/W2057216062","https://openalex.org/W2061238011","https://openalex.org/W2146901682","https://openalex.org/W2156464996","https://openalex.org/W2163861531","https://openalex.org/W2308500666","https://openalex.org/W2474019398","https://openalex.org/W2503235041","https://openalex.org/W2624630429","https://openalex.org/W2921025948","https://openalex.org/W2923771170","https://openalex.org/W2937245495","https://openalex.org/W2948287327"],"related_works":["https://openalex.org/W625706468","https://openalex.org/W2265986062","https://openalex.org/W3003558595","https://openalex.org/W2035798565","https://openalex.org/W2316908264","https://openalex.org/W2115979595","https://openalex.org/W4242511140","https://openalex.org/W1963493069","https://openalex.org/W2067995684","https://openalex.org/W2904195262"],"abstract_inverted_index":{"Novel":[0],"high":[1],"performance":[2],"computing":[3],"systems":[4,39],"that":[5,132],"feature":[6],"hybrid":[7],"architectures":[8],"require":[9],"large":[10],"scale":[11],"code":[12],"refactoring":[13],"to":[14,79,106,127,231,240],"unravel":[15],"underlying":[16],"exploitable":[17],"parallelism.":[18],"Such":[19],"redesign":[20],"can":[21,47],"often":[22],"be":[23,35],"accompanied":[24],"with":[25,170],"machine-precision":[26,57],"changes":[27],"as":[28,56,110],"the":[29,84,129,133,141,176,183,191,218,222,233,241],"order":[30],"of":[31,76,83,88,91,145,158,185,207,221,235],"computation":[32],"cannot":[33],"always":[34],"maintained.":[36],"For":[37],"chaotic":[38],"like":[40],"climate":[41,174],"models,":[42],"these":[43,146,211],"round-off":[44],"level":[45,63],"differences":[46,64],"grow":[48],"rapidly.":[49],"Systematic":[50],"errors":[51,67],"may":[52],"also":[53],"manifest":[54],"initially":[55],"differences.":[58],"Isolating":[59],"genuine":[60],"round":[61],"off":[62],"from":[65],"such":[66],"remains":[68],"a":[69,107,111,116,121,150,155,171,186,197],"challenge.":[70],"Here,":[71],"we":[72,148],"apply":[73],"two-sample":[74],"equality":[75],"distribution":[77],"tests":[78,223],"evaluate":[80,128,232],"statistical":[81],"reproducibility":[82],"ocean":[85],"model":[86,122,188,229],"component":[87,123],"US":[89],"Department":[90],"Energy's":[92],"Energy":[93],"Exascale":[94],"Earth":[95],"System":[96],"Model":[97],"(E3SM).":[98],"A":[99],"2-year":[100],"control":[101],"simulation":[102,160],"ensemble":[103,109,163,178,226],"is":[104,124,203],"compared":[105],"modified":[108],"test":[112],"case":[113],"-":[114,126,179],"after":[115],"known":[117],"non-bit-for-bit":[118,238],"change":[119,239],"in":[120,196],"introduced":[125,237],"null":[130,201],"hypothesis":[131,202],"two":[134],"ensembles":[135,209],"are":[136],"statistically":[137],"indistinguishable.":[138],"To":[139],"quantify":[140],"false":[142],"negative":[143],"rates":[144],"tests,":[147],"conduct":[149],"formal":[151],"power":[152,214],"analysis":[153,215],"using":[154,210],"targeted":[156],"suite":[157,164],"short":[159],"ensembles.":[161],"The":[162,200,213],"contains":[165],"several":[166],"perturbed":[167,208],"ensembles,":[168],"each":[169,206],"progressively":[172],"different":[173],"than":[175],"baseline":[177],"obtained":[180],"by":[181],"perturbing":[182],"magnitude":[184],"single":[187],"tuning":[189],"parameter,":[190],"Gent":[192],"and":[193],"McWilliams":[194],"\u03ba,":[195],"controlled":[198],"manner.":[199],"evaluated":[204],"for":[205,224],"tests.":[212],"informs":[216],"on":[217],"detection":[219],"limits":[220],"given":[225],"size":[227],"allowing":[228],"developers":[230],"impact":[234],"an":[236],"model.":[242]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":1},{"year":2022,"cited_by_count":2},{"year":2021,"cited_by_count":2}],"updated_date":"2026-08-01T09:00:35.917206","created_date":"2025-10-10T00:00:00"}