{"id":"https://openalex.org/W7166797490","doi":"https://doi.org/10.48550/arxiv.2606.30987","title":"Measuring Judgment Quality in Natural-Language Explanations: Evidence from Forecasting Tournaments","display_name":"Measuring Judgment Quality in Natural-Language Explanations: Evidence from Forecasting Tournaments","publication_year":2026,"publication_date":"2026-06-29","ids":{"openalex":"https://openalex.org/W7166797490","doi":"https://doi.org/10.48550/arxiv.2606.30987"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.30987","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.30987","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.30987","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5008463924","display_name":"Christopher W. Karvetski","orcid":"https://orcid.org/0000-0001-5205-7066"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Karvetski, Christopher W.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5043900893","display_name":"Sicong Huang","orcid":"https://orcid.org/0000-0003-0854-4734"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Huang, Sheldon S.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5066344269","display_name":"Simas Ku\u010dinskas","orcid":"https://orcid.org/0000-0003-4772-9424"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ku\u010dinskas, Simas","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139770419","display_name":"Nadja Flechner","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Flechner, Nadja","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139800151","display_name":"Jingyu Hu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hu, Jingyu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133493267","display_name":"Philip Tetlock","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tetlock, Philip","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5046987433","display_name":"Ezra Karger","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Karger, Ezra","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11918","display_name":"Forecasting Techniques and Applications","score":0.3059000074863434,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T11918","display_name":"Forecasting Techniques and Applications","score":0.3059000074863434,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.18440000712871552,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10315","display_name":"Decision-Making and Behavioral Economics","score":0.07540000230073929,"subfield":{"id":"https://openalex.org/subfields/1800","display_name":"General Decision Sciences"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.6450999975204468},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.5583999752998352},{"id":"https://openalex.org/keywords/probabilistic-logic","display_name":"Probabilistic logic","score":0.5412999987602234},{"id":"https://openalex.org/keywords/probabilistic-forecasting","display_name":"Probabilistic forecasting","score":0.5343000292778015},{"id":"https://openalex.org/keywords/measure","display_name":"Measure (data warehouse)","score":0.4968000054359436},{"id":"https://openalex.org/keywords/scoring-rule","display_name":"Scoring rule","score":0.46779999136924744}],"concepts":[{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.6450999975204468},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.5583999752998352},{"id":"https://openalex.org/C49937458","wikidata":"https://www.wikidata.org/wiki/Q2599292","display_name":"Probabilistic logic","level":2,"score":0.5412999987602234},{"id":"https://openalex.org/C122282355","wikidata":"https://www.wikidata.org/wiki/Q7246855","display_name":"Probabilistic forecasting","level":3,"score":0.5343000292778015},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.4968000054359436},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.48500001430511475},{"id":"https://openalex.org/C63002673","wikidata":"https://www.wikidata.org/wiki/Q2260590","display_name":"Scoring rule","level":2,"score":0.46779999136924744},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4666000008583069},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.4553000032901764},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4546999931335449},{"id":"https://openalex.org/C170061395","wikidata":"https://www.wikidata.org/wiki/Q5468164","display_name":"Forecast skill","level":2,"score":0.4009999930858612},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.3280999958515167},{"id":"https://openalex.org/C174303752","wikidata":"https://www.wikidata.org/wiki/Q3624438","display_name":"Forecast error","level":2,"score":0.30169999599456787},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.28299999237060547},{"id":"https://openalex.org/C166851805","wikidata":"https://www.wikidata.org/wiki/Q5468165","display_name":"Forecast verification","level":3,"score":0.27900001406669617},{"id":"https://openalex.org/C120954023","wikidata":"https://www.wikidata.org/wiki/Q1127277","display_name":"Consensus forecast","level":2,"score":0.27869999408721924}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.30987","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.30987","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.30987","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.30987","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/16","score":0.7097538709640503,"display_name":"Peace, Justice and strong institutions"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Decision-makers":[0],"routinely":[1],"rely":[2],"on":[3,158],"expert":[4],"judgments":[5,27],"accompanied":[6],"by":[7,52],"written":[8,179],"explanations,":[9],"yet":[10],"explanation":[11],"quality":[12,147],"is":[13,101],"difficult":[14],"to":[15,163],"measure":[16],"at":[17,74,128,134],"scale.":[18],"Forecasting":[19],"tournaments":[20],"offer":[21],"a":[22,44,58,67,170],"natural":[23],"testing":[24],"ground:":[25],"probabilistic":[26],"are":[28,124,148],"paired":[29],"with":[30,152],"natural-language":[31],"rationales":[32],"and":[33,78,132,154],"scored":[34,51],"against":[35,117],"realized":[36],"outcomes.":[37],"We":[38],"introduce":[39],"Explanation":[40],"Quality":[41],"Markers":[42],"(EQMs),":[43],"set":[45],"of":[46,61,89,120,145],"sixty":[47],"theory-guided":[48],"reasoning":[49],"patterns":[50],"large":[53],"language":[54],"models":[55],"(LLMs).":[56],"In":[57],"pre-registered":[59],"analysis":[60],"over":[62],"55,000":[63],"forecast-rationale":[64],"pairs":[65],"from":[66,178],"multiyear":[68],"forecasting":[69,121,166],"tournament,":[70],"EQMs":[71,103,123,168],"predict":[72],"accuracy":[73,153],"both":[75],"the":[76,112,125,129,135],"forecast":[77,130],"forecaster":[79,136],"levels,":[80],"consistently":[81,150],"outperforming":[82],"pre-LLM":[83],"text-analysis":[84],"methods.":[85],"More":[86],"than":[87,109,140],"90%":[88],"statistically":[90],"significant":[91],"pattern-level":[92],"EQM-accuracy":[93],"correlations":[94],"match":[95],"our":[96],"directional":[97],"hypotheses.":[98],"The":[99],"signal":[100],"asymmetric:":[102],"identify":[104],"likely":[105],"underperformers":[106],"more":[107],"reliably":[108],"they":[110],"distinguish":[111],"very":[113],"best":[114],"forecasters.":[115],"Benchmarked":[116],"traditional":[118],"indicators":[119],"skill,":[122],"strongest":[126],"predictor":[127],"level":[131],"competitive":[133],"level,":[137],"though":[138],"weaker":[139],"prior":[141],"accuracy.":[142],"Human":[143],"ratings":[144],"rationale":[146,159],"less":[149],"correlated":[151],"place":[155],"disproportionate":[156],"weight":[157],"length.":[160],"Results":[161],"transfer":[162],"an":[164],"independent":[165],"study.":[167],"provide":[169],"scalable,":[171],"interpretable":[172],"method":[173],"for":[174],"extracting":[175],"judgment-relevant":[176],"information":[177],"explanations.":[180]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-07-02T00:00:00"}