{"id":"https://openalex.org/W7136759908","doi":"https://doi.org/10.48550/arxiv.2603.13099","title":"Beyond Final Answers: CRYSTAL Benchmark for Transparent Multimodal Reasoning Evaluation","display_name":"Beyond Final Answers: CRYSTAL Benchmark for Transparent Multimodal Reasoning Evaluation","publication_year":2026,"publication_date":"2026-03-13","ids":{"openalex":"https://openalex.org/W7136759908","doi":"https://doi.org/10.48550/arxiv.2603.13099"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2603.13099","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.13099","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2603.13099","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5012883610","display_name":"Wayner Barrios","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Barrios, Wayner","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5129576055","display_name":"SouYoung Jin","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jin, SouYoung","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.1542000025510788,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.1542000025510788,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.1136000007390976,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10906","display_name":"AI-based Problem Solving and Planning","score":0.08860000222921371,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.7174000144004822},{"id":"https://openalex.org/keywords/correctness","display_name":"Correctness","score":0.6305000185966492},{"id":"https://openalex.org/keywords/cluster-analysis","display_name":"Cluster analysis","score":0.554099977016449},{"id":"https://openalex.org/keywords/pipeline","display_name":"Pipeline (software)","score":0.539900004863739},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.5011000037193298},{"id":"https://openalex.org/keywords/similarity","display_name":"Similarity (geometry)","score":0.48660001158714294},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.42800000309944153},{"id":"https://openalex.org/keywords/semantics","display_name":"Semantics (computer science)","score":0.40459999442100525},{"id":"https://openalex.org/keywords/benchmarking","display_name":"Benchmarking","score":0.37950000166893005}],"concepts":[{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.7174000144004822},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.695900022983551},{"id":"https://openalex.org/C55439883","wikidata":"https://www.wikidata.org/wiki/Q360812","display_name":"Correctness","level":2,"score":0.6305000185966492},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5892999768257141},{"id":"https://openalex.org/C73555534","wikidata":"https://www.wikidata.org/wiki/Q622825","display_name":"Cluster analysis","level":2,"score":0.554099977016449},{"id":"https://openalex.org/C43521106","wikidata":"https://www.wikidata.org/wiki/Q2165493","display_name":"Pipeline (software)","level":2,"score":0.539900004863739},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.5011000037193298},{"id":"https://openalex.org/C103278499","wikidata":"https://www.wikidata.org/wiki/Q254465","display_name":"Similarity (geometry)","level":3,"score":0.48660001158714294},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4471000134944916},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.42800000309944153},{"id":"https://openalex.org/C184337299","wikidata":"https://www.wikidata.org/wiki/Q1437428","display_name":"Semantics (computer science)","level":2,"score":0.40459999442100525},{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.37950000166893005},{"id":"https://openalex.org/C85847156","wikidata":"https://www.wikidata.org/wiki/Q59015987","display_name":"Verifiable secret sharing","level":3,"score":0.36480000615119934},{"id":"https://openalex.org/C195344581","wikidata":"https://www.wikidata.org/wiki/Q2555318","display_name":"Automated reasoning","level":2,"score":0.35040000081062317},{"id":"https://openalex.org/C177774035","wikidata":"https://www.wikidata.org/wiki/Q1246948","display_name":"Granularity","level":2,"score":0.349700003862381},{"id":"https://openalex.org/C2779346075","wikidata":"https://www.wikidata.org/wiki/Q7268763","display_name":"Quality Score","level":3,"score":0.3287000060081482},{"id":"https://openalex.org/C81669768","wikidata":"https://www.wikidata.org/wiki/Q2359161","display_name":"Precision and recall","level":2,"score":0.3221000134944916},{"id":"https://openalex.org/C42747912","wikidata":"https://www.wikidata.org/wiki/Q1048447","display_name":"Multiplicative function","level":2,"score":0.30809998512268066},{"id":"https://openalex.org/C100660578","wikidata":"https://www.wikidata.org/wiki/Q18733","display_name":"Recall","level":2,"score":0.3061999976634979},{"id":"https://openalex.org/C99844830","wikidata":"https://www.wikidata.org/wiki/Q102441924","display_name":"Scaling","level":2,"score":0.3046000003814697},{"id":"https://openalex.org/C2781020372","wikidata":"https://www.wikidata.org/wiki/Q533093","display_name":"On the fly","level":2,"score":0.30410000681877136},{"id":"https://openalex.org/C159032336","wikidata":"https://www.wikidata.org/wiki/Q2488768","display_name":"Non-monotonic logic","level":2,"score":0.29280000925064087},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.2851000130176544},{"id":"https://openalex.org/C2780513914","wikidata":"https://www.wikidata.org/wiki/Q18210350","display_name":"Bottleneck","level":2,"score":0.28139999508857727},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.27459999918937683},{"id":"https://openalex.org/C130318100","wikidata":"https://www.wikidata.org/wiki/Q2268914","display_name":"Semantic similarity","level":2,"score":0.27149999141693115},{"id":"https://openalex.org/C99498987","wikidata":"https://www.wikidata.org/wiki/Q2210247","display_name":"Noise (video)","level":3,"score":0.2606000006198883},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.2596000134944916},{"id":"https://openalex.org/C184898388","wikidata":"https://www.wikidata.org/wiki/Q1435712","display_name":"Pairwise comparison","level":2,"score":0.25940001010894775},{"id":"https://openalex.org/C127705205","wikidata":"https://www.wikidata.org/wiki/Q5748245","display_name":"Heuristics","level":2,"score":0.2542000114917755}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2603.13099","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.13099","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2603.13099","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.13099","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"We":[0,25],"introduce":[1],"CRYSTAL":[2],"(Clear":[3],"Reasoning":[4],"via":[5,38,70,165],"Yielded":[6],"Steps,":[7],"Traceability,":[8],"and":[9,36,42,73,110,148],"Logic),":[10],"a":[11,56,138,159],"diagnostic":[12],"benchmark":[13,90],"with":[14,145],"6,372":[15],"instances":[16],"that":[17,95,141],"evaluates":[18],"multimodal":[19],"reasoning":[20,50,112,153,173],"through":[21,55,75],"verifiable":[22],"intermediate":[23],"steps.":[24],"propose":[26,132],"two":[27],"complementary":[28],"metrics:":[29],"Match":[30,44,163],"F1,":[31,45],"which":[32,46,60,66,114,150],"scores":[33],"step-level":[34,146],"precision":[35],"recall":[37],"semantic":[39,71],"similarity":[40],"matching,":[41],"Ordered":[43],"further":[47],"penalizes":[48],"disordered":[49,111],"chains.":[51],"References":[52],"are":[53,67,96],"constructed":[54],"Delphi-inspired":[57],"pipeline":[58],"in":[59,113,125,162],"four":[61],"independent":[62],"MLLMs":[63],"generate":[64],"trajectories,":[65],"then":[68],"aggregated":[69],"clustering":[72],"validated":[74],"human":[76],"quality":[77],"gates.":[78],"Evaluation":[79],"of":[80,122],"20":[81],"MLLMs,":[82],"including":[83],"commercial":[84],"frontier":[85],"systems":[86],"not":[87],"used":[88],"during":[89,155],"construction,":[91],"reveals":[92],"systematic":[93],"failures":[94],"invisible":[97],"to":[98],"answer":[99,143],"accuracy:":[100],"universal":[101],"cherry-picking":[102],"(precision":[103],"far":[104],"exceeds":[105],"recall),":[106],"non-monotonic":[107],"scaling":[108],"trade-offs,":[109],"no":[115],"competitive":[116],"model":[117],"preserves":[118],"more":[119],"than":[120],"60%":[121],"matched":[123],"steps":[124],"the":[126,133],"correct":[127],"order.":[128],"Beyond":[129],"evaluation,":[130],"we":[131],"Causal":[134],"Process":[135],"Reward":[136],"(CPR),":[137],"multiplicative":[139],"reward":[140,169],"couples":[142],"correctness":[144],"alignment,":[147],"CPR-Curriculum,":[149],"progressively":[151],"increases":[152],"difficulty":[154],"training.":[156],"CPR-Curriculum":[157],"achieves":[158],"32%":[160],"improvement":[161],"F1":[164],"GRPO":[166],"where":[167],"additive":[168],"strategies":[170],"fail,":[171],"improving":[172],"without":[174],"manual":[175],"step":[176],"annotation.":[177]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-03-17T00:00:00"}