{"id":"https://openalex.org/W7166567548","doi":"https://doi.org/10.48550/arxiv.2606.27383","title":"CalBrief: A Pilot Diagnostic Benchmark for Evidence-Calibrated Scientific Briefing with Large Language Models","display_name":"CalBrief: A Pilot Diagnostic Benchmark for Evidence-Calibrated Scientific Briefing with Large Language Models","publication_year":2026,"publication_date":"2026-06-11","ids":{"openalex":"https://openalex.org/W7166567548","doi":"https://doi.org/10.48550/arxiv.2606.27383"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.27383","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.27383","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.27383","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5069101640","display_name":"Y Fu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Fu, Yu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139575423","display_name":"Yongqi Kang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kang, Yongqi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5139577353","display_name":"Yong Zhao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhao, Yong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.5587999820709229,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},"topics":[{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.5587999820709229,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.06279999762773514,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13910","display_name":"Computational and Text Analysis Methods","score":0.062199998646974564,"subfield":{"id":"https://openalex.org/subfields/3300","display_name":"General Social Sciences"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/scope","display_name":"Scope (computer science)","score":0.7397000193595886},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.704800009727478},{"id":"https://openalex.org/keywords/pipeline","display_name":"Pipeline (software)","score":0.6057999730110168},{"id":"https://openalex.org/keywords/matching","display_name":"Matching (statistics)","score":0.5390999913215637},{"id":"https://openalex.org/keywords/ambiguity","display_name":"Ambiguity","score":0.5175999999046326},{"id":"https://openalex.org/keywords/binary-number","display_name":"Binary number","score":0.4729999899864197},{"id":"https://openalex.org/keywords/benchmarking","display_name":"Benchmarking","score":0.32760000228881836}],"concepts":[{"id":"https://openalex.org/C2778012447","wikidata":"https://www.wikidata.org/wiki/Q1034415","display_name":"Scope (computer science)","level":2,"score":0.7397000193595886},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.704800009727478},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6718999743461609},{"id":"https://openalex.org/C43521106","wikidata":"https://www.wikidata.org/wiki/Q2165493","display_name":"Pipeline (software)","level":2,"score":0.6057999730110168},{"id":"https://openalex.org/C165064840","wikidata":"https://www.wikidata.org/wiki/Q1321061","display_name":"Matching (statistics)","level":2,"score":0.5390999913215637},{"id":"https://openalex.org/C2780522230","wikidata":"https://www.wikidata.org/wiki/Q1140419","display_name":"Ambiguity","level":2,"score":0.5175999999046326},{"id":"https://openalex.org/C48372109","wikidata":"https://www.wikidata.org/wiki/Q3913","display_name":"Binary number","level":2,"score":0.4729999899864197},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3970000147819519},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.390500009059906},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.38909998536109924},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.34619998931884766},{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.32760000228881836},{"id":"https://openalex.org/C2776505523","wikidata":"https://www.wikidata.org/wiki/Q4785468","display_name":"Plan (archaeology)","level":2,"score":0.2992999851703644},{"id":"https://openalex.org/C206345919","wikidata":"https://www.wikidata.org/wiki/Q20380951","display_name":"Resource (disambiguation)","level":2,"score":0.29330000281333923},{"id":"https://openalex.org/C136197465","wikidata":"https://www.wikidata.org/wiki/Q1729295","display_name":"Variety (cybernetics)","level":2,"score":0.2824000120162964},{"id":"https://openalex.org/C66905080","wikidata":"https://www.wikidata.org/wiki/Q17005494","display_name":"Binary classification","level":3,"score":0.2818000018596649},{"id":"https://openalex.org/C2778572836","wikidata":"https://www.wikidata.org/wiki/Q380933","display_name":"Space (punctuation)","level":2,"score":0.27489998936653137},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.2727000117301941},{"id":"https://openalex.org/C2779843651","wikidata":"https://www.wikidata.org/wiki/Q7390335","display_name":"SIGNAL (programming language)","level":2,"score":0.2676999866962433},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.26339998841285706},{"id":"https://openalex.org/C177212765","wikidata":"https://www.wikidata.org/wiki/Q627335","display_name":"Workflow","level":2,"score":0.262800008058548},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.25850000977516174}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.27383","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.27383","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.27383","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.27383","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Large":[0],"language":[1],"models":[2],"(LLMs)":[3],"are":[4,225],"increasingly":[5],"used":[6],"as":[7,79],"research":[8,18,238],"assistants,":[9],"yet":[10],"it":[11],"remains":[12],"unclear":[13],"whether":[14],"they":[15],"can":[16,193],"calibrate":[17],"takeaways":[19,46],"to":[20,83,147,156,172,198],"the":[21,26,142,149,178,183,209],"strength":[22,219],"and":[23,52,67,71,97,108,112,200,221,231],"scope":[24,50],"of":[25,38,61,137,141],"supporting":[27],"evidence.":[28],"We":[29,55,187],"study":[30],"evidence-calibrated":[31],"scientific":[32,64],"briefing:":[33],"given":[34],"a":[35,41,57,80,90,120],"bounded":[36],"package":[37],"related":[39],"papers,":[40],"system":[42],"should":[43,232],"generate":[44],"package-level":[45],"with":[47],"evidence":[48,65,223],"strength,":[49],"boundaries,":[51],"missing-evidence":[53],"caveats.":[54],"contribute":[56],"verified":[58],"pilot":[59],"benchmark":[60],"16":[62],"heterogeneous":[63],"packages":[66],"96":[68],"human-verified":[69],"takeaways,":[70],"we":[72,118],"use":[73],"CalBrief,":[74],"an":[75,101],"auditable":[76,222],"role/gap/strength":[77],"framework,":[78],"diagnostic":[81,122],"probe":[82],"locate":[84],"where":[85],"briefing":[86],"breaks":[87],"down.":[88],"Under":[89],"fair-schema":[91],"evaluation,":[92],"structured":[93],"organization":[94,224],"improves":[95],"role":[96],"gap":[98,144],"reasoning,":[99],"but":[100],"explicit":[102],"strength-calibration":[103],"policy":[104,185],"is":[105,145,170],"systematically":[106],"over-conservative":[107],"falls":[109],"below":[110],"majority":[111],"direct-LLM":[113],"baselines.":[114],"To":[115],"explain":[116],"why,":[117],"run":[119],"controlled":[121],"across":[123,165],"three":[124,134],"closed-model":[125],"backbones":[126],"(GPT-4o,":[127],"Claude":[128],"Sonnet,":[129],"Gemini":[130],"Flash)":[131],"that":[132,190,214],"separates":[133],"potential":[135],"causes":[136],"conservatism.":[138],"Approximately":[139],"63%":[140],"conservatism":[143],"attributable":[146,171],"expanding":[148],"label":[150],"space":[151],"from":[152,182],"binary":[153,199,206],"{moderate,":[154,158],"weak}":[155],"four-way":[157],"weak,":[159],"uncertain,":[160],"insufficient_evidence}":[161],"(p":[162],"<":[163],"0.001":[164],"all":[166],"backbones);":[167],"only":[168],"1%":[169],"gap/scope":[173],"signal":[174],"injection":[175],"(not":[176],"significant);":[177],"remaining":[179],"36%":[180],"arises":[181],"pipeline":[184],"itself.":[186],"also":[188],"find":[189],"4-way":[191],"predictions":[192],"be":[194,233],"post-hoc":[195],"collapsed":[196],"back":[197],"then":[201],"match":[202],"or":[203],"exceed":[204],"direct":[205],"prompting,":[207],"so":[208],"extra":[210],"labels":[211],"carry":[212],"information":[213],"strict":[215],"matching":[216],"hides.":[217],"Label-level":[218],"judgment":[220],"distinct":[226],"abilities":[227],"currently":[228],"in":[229],"tension,":[230],"evaluated":[234],"separately":[235],"for":[236],"LLM":[237],"assistants.":[239]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-06-30T00:00:00"}