{"id":"https://openalex.org/W7223318463","doi":"https://doi.org/10.48550/arxiv.2610.31237","title":"Can We Trust Evaluations of AI Weather Forecasts for Extremes? How Evaluation Protocols Shape Model Rankings","display_name":"Can We Trust Evaluations of AI Weather Forecasts for Extremes? How Evaluation Protocols Shape Model Rankings","publication_year":2026,"publication_date":"2026-10-22","ids":{"openalex":"https://openalex.org/W7223318463","doi":"https://doi.org/10.48550/arxiv.2610.31237"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2610.31237","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2610.31237","pdf_url":"https://arxiv.org/pdf/2610.31237","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2610.31237","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5120327079","display_name":"Tobias Biegert","orcid":"https://orcid.org/0009-0008-5623-0186","observed_orcids":["https://orcid.org/0009-0008-5623-0186"]},"institutions":[{"id":"https://openalex.org/I102335020","display_name":"Karlsruhe Institute of Technology","ror":"https://ror.org/04t3en479","country_code":"DE","type":"education","lineage":["https://openalex.org/I102335020","https://openalex.org/I1305996414"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Biegert, Tobias","raw_affiliation_strings":["Institute of Statistics Karlsruhe Institute of Technology 76185 Karlsruhe , Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Statistics Karlsruhe Institute of Technology 76185 Karlsruhe , Germany","institution_ids":["https://openalex.org/I102335020"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5085323024","display_name":"Sam Allen","orcid":"https://orcid.org/0000-0003-1971-8277","observed_orcids":["https://orcid.org/0000-0003-1971-8277"]},"institutions":[{"id":"https://openalex.org/I102335020","display_name":"Karlsruhe Institute of Technology","ror":"https://ror.org/04t3en479","country_code":"DE","type":"education","lineage":["https://openalex.org/I102335020","https://openalex.org/I1305996414"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Allen, Sam","raw_affiliation_strings":["Institute of Statistics Karlsruhe Institute of Technology 76185 Karlsruhe , Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Statistics Karlsruhe Institute of Technology 76185 Karlsruhe , Germany","institution_ids":["https://openalex.org/I102335020"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5159306917","display_name":"Zhongwei Zhang","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I102335020","display_name":"Karlsruhe Institute of Technology","ror":"https://ror.org/04t3en479","country_code":"DE","type":"education","lineage":["https://openalex.org/I102335020","https://openalex.org/I1305996414"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Zhang, Zhongwei","raw_affiliation_strings":["Institute of Statistics Karlsruhe Institute of Technology 76185 Karlsruhe , Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Statistics Karlsruhe Institute of Technology 76185 Karlsruhe , Germany","institution_ids":["https://openalex.org/I102335020"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5060759568","display_name":"Sebastian Lerch","orcid":"https://orcid.org/0000-0002-3467-4375","observed_orcids":["https://orcid.org/0000-0002-3467-4375"]},"institutions":[{"id":"https://openalex.org/I161103922","display_name":"Philipps University of Marburg","ror":"https://ror.org/01rdrb571","country_code":"DE","type":"education","lineage":["https://openalex.org/I161103922"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Lerch, Sebastian","raw_affiliation_strings":["Dept. of Mathematics and Computer Science Marburg University 35032 Marburg , Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Dept. of Mathematics and Computer Science Marburg University 35032 Marburg , Germany","institution_ids":["https://openalex.org/I161103922"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10466","display_name":"Meteorological Phenomena and Simulations","score":0.9883000254631042,"subfield":{"id":"https://openalex.org/subfields/1902","display_name":"Atmospheric Science"},"field":{"id":"https://openalex.org/fields/19","display_name":"Earth and Planetary Sciences"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10466","display_name":"Meteorological Phenomena and Simulations","score":0.9883000254631042,"subfield":{"id":"https://openalex.org/subfields/1902","display_name":"Atmospheric Science"},"field":{"id":"https://openalex.org/fields/19","display_name":"Earth and Planetary Sciences"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10029","display_name":"Climate variability and models","score":0.003000000026077032,"subfield":{"id":"https://openalex.org/subfields/2306","display_name":"Global and Planetary Change"},"field":{"id":"https://openalex.org/fields/23","display_name":"Environmental Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11918","display_name":"Forecasting Techniques and Applications","score":0.0017000000225380063,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/extreme-weather-events","display_name":"extreme weather events","score":0.9440000057220459},{"id":"https://openalex.org/keywords/forecast-evaluation-metrics","display_name":"forecast evaluation metrics","score":0.8240000009536743},{"id":"https://openalex.org/keywords/model-ranking","display_name":"model ranking","score":0.6309999823570251},{"id":"https://openalex.org/keywords/contingency-table-analysis","display_name":"contingency table analysis","score":0.7770000100135803},{"id":"https://openalex.org/keywords/calibration-error","display_name":"calibration error","score":0.9210000038146973},{"id":"https://openalex.org/keywords/discrimination-statistics","display_name":"discrimination (statistics)","score":0.47999998927116394},{"id":"https://openalex.org/keywords/operational-forecasting","display_name":"operational forecasting","score":0.3959999978542328}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5127000212669373},{"id":"https://openalex.org/C91998498","wikidata":"https://www.wikidata.org/wiki/Q1541178","display_name":"Contingency table","level":2,"score":0.42480000853538513},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.4246000051498413},{"id":"https://openalex.org/C205537798","wikidata":"https://www.wikidata.org/wiki/Q1277161","display_name":"Extreme weather","level":3,"score":0.40220001339912415},{"id":"https://openalex.org/C170061395","wikidata":"https://www.wikidata.org/wiki/Q5468164","display_name":"Forecast skill","level":2,"score":0.39649999141693115},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3765000104904175},{"id":"https://openalex.org/C149629883","wikidata":"https://www.wikidata.org/wiki/Q660926","display_name":"Fraction (chemistry)","level":2,"score":0.3483000099658966},{"id":"https://openalex.org/C45804977","wikidata":"https://www.wikidata.org/wiki/Q7239673","display_name":"Predictive modelling","level":2,"score":0.3479999899864197},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.32269999384880066},{"id":"https://openalex.org/C45235069","wikidata":"https://www.wikidata.org/wiki/Q278425","display_name":"Table (database)","level":2,"score":0.3070000112056732},{"id":"https://openalex.org/C12725497","wikidata":"https://www.wikidata.org/wiki/Q810247","display_name":"Baseline (sea)","level":2,"score":0.2849000096321106},{"id":"https://openalex.org/C63002673","wikidata":"https://www.wikidata.org/wiki/Q2260590","display_name":"Scoring rule","level":2,"score":0.2831999957561493},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.26489999890327454}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2610.31237","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2610.31237","pdf_url":"https://arxiv.org/pdf/2610.31237","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2610.31237","doi":"https://doi.org/10.48550/arxiv.2610.31237","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2610.31237","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2610.31237","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2610.31237","pdf_url":"https://arxiv.org/pdf/2610.31237","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[{"display_name":"Climate action","score":0.821399986743927,"id":"https://metadata.un.org/sdg/13"}],"study_designs":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320328198","display_name":"Vector Stiftung","ror":"https://ror.org/05ymr3m21"},{"id":"https://openalex.org/F4320330412","display_name":"Scheme for Promotion of Academic and Research Collaboration","ror":null}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7223318463.pdf","grobid_xml":"https://content.openalex.org/works/W7223318463.grobid-xml"},"referenced_works_count":26,"referenced_works":["https://openalex.org/W2075965721","https://openalex.org/W2216786191","https://openalex.org/W2944472092","https://openalex.org/W2963870508","https://openalex.org/W2966241529","https://openalex.org/W2971435259","https://openalex.org/W3025949386","https://openalex.org/W3047801607","https://openalex.org/W3192681150","https://openalex.org/W3210061498","https://openalex.org/W4383218913","https://openalex.org/W4388654737","https://openalex.org/W4388728292","https://openalex.org/W4391651925","https://openalex.org/W4399401448","https://openalex.org/W4399668152","https://openalex.org/W4399779937","https://openalex.org/W4404143263","https://openalex.org/W4404826731","https://openalex.org/W4410551352","https://openalex.org/W4410553701","https://openalex.org/W7102781402","https://openalex.org/W7128953577","https://openalex.org/W7133763938","https://openalex.org/W7158534923","https://openalex.org/W7165692131"],"related_works":["https://openalex.org/W7165692131","https://openalex.org/W7202269652","https://openalex.org/W7160118411","https://openalex.org/W7103104626","https://openalex.org/W4392758858","https://openalex.org/W7160254805","https://openalex.org/W7222544087","https://openalex.org/W7170166157","https://openalex.org/W7169789343","https://openalex.org/W4416051540"],"abstract_inverted_index":{"AI":[0,54,98,115,191],"weather":[1,34,57],"models":[2,8,99,116],"perform":[3,87,101],"competitively":[4],"with":[5],"traditional":[6],"physics-based":[7,56,83,133],"at":[9],"a":[10,63],"fraction":[11],"of":[12,78,120,184],"the":[13,48,76,81,118,132,159,174,178,182,185],"computational":[14],"cost,":[15],"and":[16,36,55,139,180],"they":[17],"are":[18,40,72,126,142],"thus":[19],"already":[20],"used":[21,188],"operationally.":[22],"However,":[23],"studies":[24,39],"reach":[25],"conflicting":[26],"conclusions":[27],"about":[28],"their":[29,124],"ability":[30],"to":[31,75,86,93,189],"forecast":[32],"extreme":[33,60,95],"events,":[35,96],"comparisons":[37],"across":[38],"complicated":[41],"by":[42,129,144],"differing":[43],"evaluation":[44,79,90,148,186],"protocols.":[45],"We":[46],"discuss":[47],"challenges":[49],"that":[50,69,114,123,158],"arise":[51,162],"when":[52,89,103],"comparing":[53],"forecasts":[58,135],"for":[59],"events.":[61],"Within":[62],"common":[64],"experimental":[65],"design,":[66],"we":[67],"find":[68],"model":[70,84,134],"rankings":[71],"highly":[73],"sensitive":[74],"choice":[77],"metrics:":[80],"considered":[82],"tends":[85],"best":[88],"is":[91],"restricted":[92],"observed":[94],"whereas":[97],"generally":[100,127],"better":[102],"evaluated":[104],"using":[105],"weighted":[106],"scoring":[107],"functions.":[108],"A":[109],"contingency":[110],"table":[111],"analysis":[112],"shows":[113],"underpredict":[117],"frequency":[119],"extremes,":[121],"so":[122],"scores":[125,141],"dominated":[128,143],"misses,":[130],"while":[131],"extremes":[136],"more":[137],"frequently":[138],"its":[140],"false":[145],"alarms.":[146],"Different":[147],"metrics":[149,187],"penalise":[150],"these":[151],"errors":[152],"differently.":[153],"Further":[154],"score":[155,160],"decompositions":[156],"confirm":[157],"differences":[161],"primarily":[163],"from":[164],"miscalibration":[165],"rather":[166],"than":[167],"discrimination":[168],"ability.":[169],"This":[170],"sheds":[171],"light":[172],"on":[173],"contrasting":[175],"results":[176],"in":[177],"literature,":[179],"highlights":[181],"importance":[183],"evaluate":[190],"models.":[192]},"counts_by_year":[],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2026-10-24T00:00:00"}