{"id":"https://openalex.org/W4415540927","doi":"https://doi.org/10.1145/3746027.3758308","title":"SHALE: A Scalable Benchmark for Fine-grained Hallucination Evaluation in LVLMs","display_name":"SHALE: A Scalable Benchmark for Fine-grained Hallucination Evaluation in LVLMs","publication_year":2025,"publication_date":"2025-10-25","ids":{"openalex":"https://openalex.org/W4415540927","doi":"https://doi.org/10.1145/3746027.3758308"},"language":null,"primary_location":{"id":"doi:10.1145/3746027.3758308","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3746027.3758308","pdf_url":null,"source":null,"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 33rd ACM International Conference on Multimedia","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3746027.3758308","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100945283","display_name":"Bei Yan","orcid":null},"institutions":[{"id":"https://openalex.org/I4210090176","display_name":"Institute of Computing Technology","ror":"https://ror.org/0090r4d87","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210090176"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Bei Yan","raw_affiliation_strings":["Key Laboratory of AI Safety of CAS, Institute of Computing Technology, Chinese Academy of Sciences (CAS), Beijing, China and University of Chinese Academy of Sciences, Beijing, China"],"affiliations":[{"raw_affiliation_string":"Key Laboratory of AI Safety of CAS, Institute of Computing Technology, Chinese Academy of Sciences (CAS), Beijing, China and University of Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I4210090176"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5084317062","display_name":"Zhiyuan Chen","orcid":null},"institutions":[{"id":"https://openalex.org/I4210090176","display_name":"Institute of Computing Technology","ror":"https://ror.org/0090r4d87","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210090176"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhiyuan Chen","raw_affiliation_strings":["Key Laboratory of AI Safety of CAS, Institute of Computing Technology, Chinese Academy of Sciences (CAS), Beijing, China and University of Chinese Academy of Sciences, Beijing, China"],"affiliations":[{"raw_affiliation_string":"Key Laboratory of AI Safety of CAS, Institute of Computing Technology, Chinese Academy of Sciences (CAS), Beijing, China and University of Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I4210090176"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5069644636","display_name":"Yuecong Min","orcid":"https://orcid.org/0000-0002-0696-2468"},"institutions":[{"id":"https://openalex.org/I19820366","display_name":"Chinese Academy of Sciences","ror":"https://ror.org/034t30j35","country_code":"CN","type":"government","lineage":["https://openalex.org/I19820366"]},{"id":"https://openalex.org/I4210090176","display_name":"Institute of Computing Technology","ror":"https://ror.org/0090r4d87","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210090176"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yuecong Min","raw_affiliation_strings":["Key Laboratory of AI Safety of CAS, Institute of Computing Technology, Chinese Academy of Sciences (CAS), Beijing, China"],"affiliations":[{"raw_affiliation_string":"Key Laboratory of AI Safety of CAS, Institute of Computing Technology, Chinese Academy of Sciences (CAS), Beijing, China","institution_ids":["https://openalex.org/I4210090176","https://openalex.org/I19820366"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5010618416","display_name":"Jie Zhang","orcid":"https://orcid.org/0000-0002-8899-3996"},"institutions":[{"id":"https://openalex.org/I19820366","display_name":"Chinese Academy of Sciences","ror":"https://ror.org/034t30j35","country_code":"CN","type":"government","lineage":["https://openalex.org/I19820366"]},{"id":"https://openalex.org/I4210090176","display_name":"Institute of Computing Technology","ror":"https://ror.org/0090r4d87","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210090176"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jie Zhang","raw_affiliation_strings":["Key Laboratory of AI Safety of CAS, Institute of Computing Technology, Chinese Academy of Sciences (CAS), Beijing, China"],"affiliations":[{"raw_affiliation_string":"Key Laboratory of AI Safety of CAS, Institute of Computing Technology, Chinese Academy of Sciences (CAS), Beijing, China","institution_ids":["https://openalex.org/I4210090176","https://openalex.org/I19820366"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5111350279","display_name":"Jiahao Wang","orcid":"https://orcid.org/0000-0002-6657-6403"},"institutions":[{"id":"https://openalex.org/I2250955327","display_name":"Huawei Technologies (China)","ror":"https://ror.org/00cmhce21","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250955327"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jiahao Wang","raw_affiliation_strings":["Trustworthy Technology and Engineering Laboratory, Huawei, Shenzhen, China"],"affiliations":[{"raw_affiliation_string":"Trustworthy Technology and Engineering Laboratory, Huawei, Shenzhen, China","institution_ids":["https://openalex.org/I2250955327"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5039788422","display_name":"Xiaozhen Wang","orcid":"https://orcid.org/0009-0000-8457-2636"},"institutions":[{"id":"https://openalex.org/I2250955327","display_name":"Huawei Technologies (China)","ror":"https://ror.org/00cmhce21","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250955327"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xiaozhen Wang","raw_affiliation_strings":["Trustworthy Technology and Engineering Laboratory, Huawei, Shenzhen, China"],"affiliations":[{"raw_affiliation_string":"Trustworthy Technology and Engineering Laboratory, Huawei, Shenzhen, China","institution_ids":["https://openalex.org/I2250955327"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5050297728","display_name":"Shiguang Shan","orcid":"https://orcid.org/0000-0002-8348-392X"},"institutions":[{"id":"https://openalex.org/I19820366","display_name":"Chinese Academy of Sciences","ror":"https://ror.org/034t30j35","country_code":"CN","type":"government","lineage":["https://openalex.org/I19820366"]},{"id":"https://openalex.org/I4210090176","display_name":"Institute of Computing Technology","ror":"https://ror.org/0090r4d87","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210090176"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Shiguang Shan","raw_affiliation_strings":["Key Laboratory of AI Safety of CAS, Institute of Computing Technology, Chinese Academy of Sciences (CAS), Beijing, China"],"affiliations":[{"raw_affiliation_string":"Key Laboratory of AI Safety of CAS, Institute of Computing Technology, Chinese Academy of Sciences (CAS), Beijing, China","institution_ids":["https://openalex.org/I4210090176","https://openalex.org/I19820366"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":7,"corresponding_author_ids":["https://openalex.org/A5100945283"],"corresponding_institution_ids":["https://openalex.org/I4210090176"],"apc_list":null,"apc_paid":null,"fwci":1.4344,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":{"value":0.85585665,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":96,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"13442","last_page":"13449"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10241","display_name":"Functional Brain Connectivity Studies","score":0.9904999732971191,"subfield":{"id":"https://openalex.org/subfields/2805","display_name":"Cognitive Neuroscience"},"field":{"id":"https://openalex.org/fields/28","display_name":"Neuroscience"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},"topics":[{"id":"https://openalex.org/T10241","display_name":"Functional Brain Connectivity Studies","score":0.9904999732971191,"subfield":{"id":"https://openalex.org/subfields/2805","display_name":"Cognitive Neuroscience"},"field":{"id":"https://openalex.org/fields/28","display_name":"Neuroscience"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},{"id":"https://openalex.org/T12536","display_name":"Topological and Geometric Data Analysis","score":0.9677000045776367,"subfield":{"id":"https://openalex.org/subfields/1703","display_name":"Computational Theory and Mathematics"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10094","display_name":"Epilepsy research and treatment","score":0.9585999846458435,"subfield":{"id":"https://openalex.org/subfields/2738","display_name":"Psychiatry and Mental health"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.6603999733924866},{"id":"https://openalex.org/keywords/pipeline","display_name":"Pipeline (software)","score":0.5985999703407288},{"id":"https://openalex.org/keywords/scalability","display_name":"Scalability","score":0.5633000135421753},{"id":"https://openalex.org/keywords/categorization","display_name":"Categorization","score":0.4950000047683716},{"id":"https://openalex.org/keywords/generalization","display_name":"Generalization","score":0.454800009727478},{"id":"https://openalex.org/keywords/perception","display_name":"Perception","score":0.41589999198913574},{"id":"https://openalex.org/keywords/visual-hallucination","display_name":"Visual Hallucination","score":0.3935000002384186},{"id":"https://openalex.org/keywords/construct","display_name":"Construct (python library)","score":0.3905999958515167}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6621999740600586},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.6603999733924866},{"id":"https://openalex.org/C43521106","wikidata":"https://www.wikidata.org/wiki/Q2165493","display_name":"Pipeline (software)","level":2,"score":0.5985999703407288},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5722000002861023},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.5633000135421753},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.531000018119812},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.49790000915527344},{"id":"https://openalex.org/C94124525","wikidata":"https://www.wikidata.org/wiki/Q912550","display_name":"Categorization","level":2,"score":0.4950000047683716},{"id":"https://openalex.org/C177148314","wikidata":"https://www.wikidata.org/wiki/Q170084","display_name":"Generalization","level":2,"score":0.454800009727478},{"id":"https://openalex.org/C26760741","wikidata":"https://www.wikidata.org/wiki/Q160402","display_name":"Perception","level":2,"score":0.41589999198913574},{"id":"https://openalex.org/C2908998935","wikidata":"https://www.wikidata.org/wiki/Q130741","display_name":"Visual Hallucination","level":2,"score":0.3935000002384186},{"id":"https://openalex.org/C2780801425","wikidata":"https://www.wikidata.org/wiki/Q5164392","display_name":"Construct (python library)","level":2,"score":0.3905999958515167},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.3596000075340271},{"id":"https://openalex.org/C21200559","wikidata":"https://www.wikidata.org/wiki/Q7451068","display_name":"Sensitivity (control systems)","level":2,"score":0.3400000035762787},{"id":"https://openalex.org/C97931131","wikidata":"https://www.wikidata.org/wiki/Q5282087","display_name":"Discriminative model","level":2,"score":0.302700012922287},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.29100000858306885},{"id":"https://openalex.org/C178253425","wikidata":"https://www.wikidata.org/wiki/Q162668","display_name":"Visual perception","level":3,"score":0.28769999742507935},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.2824000120162964},{"id":"https://openalex.org/C99498987","wikidata":"https://www.wikidata.org/wiki/Q2210247","display_name":"Noise (video)","level":3,"score":0.2793000042438507},{"id":"https://openalex.org/C2780719617","wikidata":"https://www.wikidata.org/wiki/Q1030752","display_name":"Salient","level":2,"score":0.27300000190734863},{"id":"https://openalex.org/C171686336","wikidata":"https://www.wikidata.org/wiki/Q3532085","display_name":"Topic model","level":2,"score":0.26409998536109924},{"id":"https://openalex.org/C61797465","wikidata":"https://www.wikidata.org/wiki/Q1188986","display_name":"Term (time)","level":2,"score":0.2612999975681305},{"id":"https://openalex.org/C95713431","wikidata":"https://www.wikidata.org/wiki/Q631425","display_name":"Vulnerability (computing)","level":2,"score":0.2563999891281128},{"id":"https://openalex.org/C36464697","wikidata":"https://www.wikidata.org/wiki/Q451553","display_name":"Visualization","level":2,"score":0.2563000023365021},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.25519999861717224},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.250900000333786}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3746027.3758308","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3746027.3758308","pdf_url":null,"source":null,"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 33rd ACM International Conference on Multimedia","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3746027.3758308","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3746027.3758308","pdf_url":null,"source":null,"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 33rd ACM International Conference on Multimedia","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":3,"referenced_works":["https://openalex.org/W4309674289","https://openalex.org/W4388725043","https://openalex.org/W4404534210"],"related_works":[],"abstract_inverted_index":{"Despite":[0],"rapid":[1],"advances,":[2],"Large":[3],"Vision-Language":[4],"Models":[5],"(LVLMs)":[6],"still":[7],"suffer":[8],"from":[9],"hallucinations,":[10,27],"i.e.,":[11],"generating":[12],"content":[13],"inconsistent":[14],"with":[15,93],"input":[16,94],"or":[17,55],"established":[18],"world":[19],"knowledge,":[20],"which":[21],"correspond":[22],"to":[23,96,113,166],"faithfulness":[24,33,116,139],"and":[25,42,63,81,117,140,149,163],"factuality":[26,118,161],"respectively.":[28],"Prior":[29],"studies":[30],"primarily":[31],"evaluate":[32],"hallucination":[34,90,123],"at":[35],"a":[36,88,107,121],"rather":[37],"coarse":[38],"level":[39],"(e.g.,":[40],"object-level)":[41],"lack":[43],"fine-grained":[44,122],"analysis.":[45],"Additionally,":[46],"existing":[47],"benchmarks":[48],"often":[49],"rely":[50],"on":[51,154],"costly":[52],"manual":[53],"curation":[54],"reused":[56],"public":[57],"datasets,":[58],"raising":[59],"concerns":[60],"about":[61],"scalability":[62],"data":[64,74],"leakage.":[65],"To":[66],"address":[67],"these":[68,102],"limitations,":[69],"we":[70,104],"propose":[71],"an":[72],"automated":[73],"construction":[75],"pipeline":[76],"that":[77],"produces":[78],"scalable,":[79],"controllable,":[80],"diverse":[82],"evaluation":[83],"data.":[84],"We":[85],"also":[86],"design":[87],"hierarchical":[89],"induction":[91],"framework":[92],"perturbations":[95],"simulate":[97],"realistic":[98],"noisy":[99,150],"scenarios.":[100,151],"Integrating":[101],"designs,":[103],"construct":[105],"SHALE,":[106],"Scalable":[108],"HALlucination":[109],"Evaluation":[110],"benchmark":[111],"designed":[112],"assess":[114],"both":[115,147],"hallucinations":[119,162],"via":[120],"categorization":[124],"scheme.":[125],"SHALE":[126],"comprises":[127],"over":[128,155],"30K":[129],"image-instruction":[130],"pairs":[131],"spanning":[132],"12":[133],"representative":[134],"visual":[135],"perception":[136],"aspects":[137],"for":[138,144],"6":[141],"knowledge":[142],"domains":[143],"factuality,":[145],"considering":[146],"clean":[148],"Extensive":[152],"experiments":[153],"20":[156],"mainstream":[157],"LVLMs":[158],"reveal":[159],"significant":[160],"high":[164],"sensitivity":[165],"semantic":[167],"perturbations.":[168]},"counts_by_year":[{"year":2026,"cited_by_count":1}],"updated_date":"2026-03-27T05:58:40.876381","created_date":"2025-10-25T00:00:00"}