{"id":"https://openalex.org/W7160595366","doi":"https://doi.org/10.48550/arxiv.2605.05573","title":"AstroAlertBench: Evaluating the Accuracy, Reasoning, and Honesty of Multimodal LLMs in Astronomical Classification","display_name":"AstroAlertBench: Evaluating the Accuracy, Reasoning, and Honesty of Multimodal LLMs in Astronomical Classification","publication_year":2026,"publication_date":"2026-05-07","ids":{"openalex":"https://openalex.org/W7160595366","doi":"https://doi.org/10.48550/arxiv.2605.05573"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.05573","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.05573","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.05573","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5135658361","display_name":"Claire Chen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Claire","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133575979","display_name":"Jiabao Sean Xiao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xiao, Jiabao Sean","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5061588035","display_name":"Shuze Liu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Shuze Daniel","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5041233493","display_name":"Facundo P\u00e9rez Paolino","orcid":"https://orcid.org/0000-0002-4128-7867"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Paolino, Facundo Perez","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135708621","display_name":"Luke Handley","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Handley, Luke","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135645425","display_name":"Theophile Jegou du Laz","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Laz, Theophile Jegou du","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5046637824","display_name":"Ricky Nilsson","orcid":"https://orcid.org/0000-0002-5408-4954"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Nilsson, Ricky","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5121143133","display_name":"Alice Zou","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zou, Alice","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135657555","display_name":"Matthew Graham","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Graham, Matthew","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5135687760","display_name":"Ashish Mahabal","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mahabal, Ashish","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.04960000142455101,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.04960000142455101,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12836","display_name":"History and Developments in Astronomy","score":0.04780000075697899,"subfield":{"id":"https://openalex.org/subfields/3103","display_name":"Astronomy and Astrophysics"},"field":{"id":"https://openalex.org/fields/31","display_name":"Physics and Astronomy"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10799","display_name":"Data Visualization and Analytics","score":0.019999999552965164,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.7580000162124634},{"id":"https://openalex.org/keywords/bottleneck","display_name":"Bottleneck","score":0.6654999852180481},{"id":"https://openalex.org/keywords/event","display_name":"Event (particle physics)","score":0.5410000085830688},{"id":"https://openalex.org/keywords/consistency","display_name":"Consistency (knowledge bases)","score":0.4957999885082245},{"id":"https://openalex.org/keywords/metadata","display_name":"Metadata","score":0.4514000117778778},{"id":"https://openalex.org/keywords/visualization","display_name":"Visualization","score":0.3555999994277954}],"concepts":[{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.7580000162124634},{"id":"https://openalex.org/C2780513914","wikidata":"https://www.wikidata.org/wiki/Q18210350","display_name":"Bottleneck","level":2,"score":0.6654999852180481},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5877000093460083},{"id":"https://openalex.org/C2779662365","wikidata":"https://www.wikidata.org/wiki/Q5416694","display_name":"Event (particle physics)","level":2,"score":0.5410000085830688},{"id":"https://openalex.org/C2776436953","wikidata":"https://www.wikidata.org/wiki/Q5163215","display_name":"Consistency (knowledge bases)","level":2,"score":0.4957999885082245},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.46219998598098755},{"id":"https://openalex.org/C93518851","wikidata":"https://www.wikidata.org/wiki/Q180160","display_name":"Metadata","level":2,"score":0.4514000117778778},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.427700012922287},{"id":"https://openalex.org/C36464697","wikidata":"https://www.wikidata.org/wiki/Q451553","display_name":"Visualization","level":2,"score":0.3555999994277954},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3431999981403351},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.3260999917984009},{"id":"https://openalex.org/C2780385302","wikidata":"https://www.wikidata.org/wiki/Q367158","display_name":"Protocol (science)","level":3,"score":0.32350000739097595},{"id":"https://openalex.org/C2778571376","wikidata":"https://www.wikidata.org/wiki/Q1355821","display_name":"Frontier","level":2,"score":0.2996000051498413},{"id":"https://openalex.org/C98184364","wikidata":"https://www.wikidata.org/wiki/Q1780131","display_name":"Argument (complex analysis)","level":2,"score":0.27959999442100525},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.271699994802475},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.2653000056743622},{"id":"https://openalex.org/C2776834974","wikidata":"https://www.wikidata.org/wiki/Q672021","display_name":"Large Synoptic Survey Telescope","level":3,"score":0.2547000050544739}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.05573","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.05573","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.05573","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.05573","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"display_name":"Partnerships for the goals","id":"https://metadata.un.org/sdg/17","score":0.4148195683956146}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Modern":[0],"astronomical":[1,60,104,174],"observatories":[2],"generate":[3],"a":[4,11,50,64,80,93,147,153,158,167],"massive":[5],"volume":[6],"of":[7,83],"multimodal":[8,19,52],"data,":[9],"creating":[10],"critical":[12],"bottleneck":[13],"for":[14,169],"expert":[15],"human":[16],"review.":[17],"While":[18],"large":[20],"language":[21],"models":[22],"(LLMs)":[23],"have":[24],"shown":[25],"promise":[26],"in":[27,59],"interpreting":[28],"complex":[29],"visual":[30,119],"and":[31,72,114,172],"textual":[32],"inputs,":[33],"their":[34],"ability":[35,137],"to":[36,55,101,138,160],"perform":[37],"specialized":[38],"scientific":[39,70],"classification":[40,74],"while":[41],"providing":[42],"interpretable":[43,173],"reasoning":[44],"remains":[45],"understudied.":[46],"We":[47,78,150],"introduce":[48],"AstroAlertBench,":[49],"comprehensive":[51],"benchmark":[53,110],"designed":[54],"evaluate":[56],"LLM":[57],"performance":[58],"event":[61],"review":[62],"along":[63],"three-stage":[65],"logical":[66],"chain:":[67],"metadata":[68],"grounding,":[69],"reasoning,":[71,141],"hierarchical":[73],"over":[75],"five":[76],"categories.":[77],"use":[79],"pilot":[81],"sample":[82],"1,500":[84],"real-world":[85,148],"alerts":[86],"from":[87],"the":[88,98,136],"Zwicky":[89],"Transient":[90],"Facility":[91],"(ZTF),":[92],"wide-field":[94],"survey":[95],"that":[96,117,124],"scans":[97],"northern":[99],"sky":[100],"detect":[102],"transient":[103],"events.":[105],"On":[106],"this":[107],"dataset,":[108],"we":[109],"13":[111],"frontier":[112],"closed-source":[113],"open-weight":[115],"LLMs":[116],"support":[118],"input.":[120],"Our":[121],"results":[122],"reveal":[123],"high":[125],"accuracy":[126],"does":[127],"not":[128],"always":[129],"align":[130],"with":[131],"model":[132],"``honesty,''":[133],"defined":[134],"as":[135,146,157],"self-evaluate":[139],"its":[140,144],"which":[142],"affects":[143],"reliability":[145],"assistant.":[149],"further":[151],"initialize":[152],"human-in-the-loop":[154],"evaluation":[155],"protocol":[156],"precursor":[159],"future":[161],"community-scale":[162],"participation.":[163],"Together,":[164],"AstroAlertBench":[165],"provides":[166],"framework":[168],"developing":[170],"calibrated":[171],"assistants.":[175]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-05-09T00:00:00"}