{"id":"https://openalex.org/W7147267944","doi":"https://doi.org/10.48550/arxiv.2603.28590","title":"MonitorBench: A Comprehensive Benchmark for Chain-of-Thought Monitorability in Large Language Models","display_name":"MonitorBench: A Comprehensive Benchmark for Chain-of-Thought Monitorability in Large Language Models","publication_year":2026,"publication_date":"2026-03-30","ids":{"openalex":"https://openalex.org/W7147267944","doi":"https://doi.org/10.48550/arxiv.2603.28590"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2603.28590","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.28590","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2603.28590","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5132695381","display_name":"Han. Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Wang, Han","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5132683123","display_name":"Yifan Sun","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sun, Yifan","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5132642324","display_name":"Brian Ko","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ko, Brian","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5132710119","display_name":"Mann Talati","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Talati, Mann","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5012740636","display_name":"Jiawen Gong","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gong, Jiawen","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5132545904","display_name":"Zimeng Li","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Zimeng","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5132580588","display_name":"Naicheng Yu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yu, Naicheng","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5055770607","display_name":"Xucheng Yu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yu, Xucheng","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5132677933","display_name":"Wei Shen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shen, Wei","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5079986434","display_name":"Vedant Jolly","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jolly, Vedant","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5132680834","display_name":"Huan Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Huan","raw_affiliation_strings":[],"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":11,"corresponding_author_ids":["https://openalex.org/A5132695381"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.21070000529289246,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.21070000529289246,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.14880000054836273,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.1307000070810318,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.8734999895095825},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.6597999930381775},{"id":"https://openalex.org/keywords/empirical-research","display_name":"Empirical research","score":0.3716000020503998},{"id":"https://openalex.org/keywords/visual-reasoning","display_name":"Visual reasoning","score":0.36399999260902405},{"id":"https://openalex.org/keywords/test","display_name":"Test (biology)","score":0.2842000126838684}],"concepts":[{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.8734999895095825},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.6597999930381775},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6520000100135803},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.46369999647140503},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4507000148296356},{"id":"https://openalex.org/C120936955","wikidata":"https://www.wikidata.org/wiki/Q2155640","display_name":"Empirical research","level":2,"score":0.3716000020503998},{"id":"https://openalex.org/C2777508537","wikidata":"https://www.wikidata.org/wiki/Q7936620","display_name":"Visual reasoning","level":2,"score":0.36399999260902405},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.2842000126838684},{"id":"https://openalex.org/C169903167","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Test set","level":2,"score":0.2759999930858612},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.27399998903274536},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.2612000107765198},{"id":"https://openalex.org/C168167062","wikidata":"https://www.wikidata.org/wiki/Q1117970","display_name":"Component (thermodynamics)","level":2,"score":0.25589999556541443}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2603.28590","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.28590","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2603.28590","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.28590","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/16","display_name":"Peace, Justice and strong institutions","score":0.5078826546669006}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Large":[0],"language":[1],"models":[2],"(LLMs)":[3],"can":[4,103,126,171],"generate":[5],"chains":[6],"of":[7,84],"thought":[8],"(CoTs)":[9],"that":[10,139,187],"are":[11],"not":[12,189],"always":[13],"causally":[14],"responsible":[15],"for":[16,55,72,202],"their":[17],"final":[18,157],"outputs.":[19],"When":[20],"such":[21],"a":[22,49,69,81,200],"mismatch":[23],"occurs,":[24],"the":[25,31,38,43,108,145,149,156,194],"CoT":[26,45,58,74,124,140],"no":[27],"longer":[28],"faithfully":[29],"reflects":[30],"actual":[32],"reasons":[33],"(i.e.,":[34],"decision-critical":[35,91,146,195],"factors)":[36],"driving":[37,110],"model's":[39],"behavior,":[40],"leading":[41],"to":[42,99,106,118,122,163,182],"reduced":[44],"monitorability":[46,59,75,125,141,174,178,212],"problem.":[47],"However,":[48],"comprehensive":[50],"and":[51,113,214],"fully":[52],"open-source":[53],"benchmark":[54,71],"thoroughly":[56],"evaluating":[57,73,206],"remains":[60],"lacking.":[61],"To":[62],"address":[63],"this":[64],"gap,":[65],"we":[66],"propose":[67],"MonitorBench,":[68],"systematic":[70],"in":[76,184],"LLMs.":[77],"MonitorBench":[78,198],"provides:":[79],"(1)":[80],"diverse":[82],"set":[83],"1,514":[85],"test":[86],"instances":[87],"with":[88,135,177],"carefully":[89],"designed":[90],"factors":[92,109,147],"across":[93,131],"19":[94],"tasks":[95,186],"spanning":[96],"7":[97],"categories":[98],"characterize":[100],"\\textit{when}":[101],"CoTs":[102],"be":[104,127],"used":[105],"monitor":[107],"LLM":[111],"behavior;":[112],"(2)":[114],"two":[115],"stress-test":[116,211],"settings":[117],"quantify":[119],"\\textit{the":[120],"extent":[121],"which}":[123],"degraded.":[128],"Extensive":[129],"experiments":[130],"multiple":[132],"popular":[133],"LLMs":[134,161,170],"varying":[136],"capabilities":[137],"show":[138],"is":[142,221],"higher":[143],"when":[144],"shape":[148],"intermediate":[150],"reasoning":[151,192],"process":[152],"without":[153],"merely":[154],"influencing":[155],"answer.":[158],"More":[159],"capable":[160],"tend":[162],"exhibit":[164],"lower":[165],"monitorability.":[166],"And":[167],"all":[168],"evaluated":[169],"intentionally":[172],"reduce":[173],"under":[175],"stress-tests,":[176],"dropping":[179],"by":[180],"up":[181],"30\\%":[183],"some":[185],"do":[188],"require":[190],"structural":[191],"over":[193],"factors.":[196],"Overall,":[197],"provides":[199],"basis":[201],"further":[203],"research":[204],"on":[205],"future":[207],"LLMs,":[208],"studying":[209],"advanced":[210],"techniques,":[213],"developing":[215],"new":[216],"monitoring":[217],"approaches.":[218],"The":[219],"code":[220],"available":[222],"at":[223],"https://github.com/ASTRAL-Group/MonitorBench.":[224]},"counts_by_year":[],"updated_date":"2026-04-04T06:10:10.580331","created_date":"2026-04-02T00:00:00"}