{"id":"https://openalex.org/W4414902532","doi":"https://doi.org/10.1109/iccv51701.2025.00365","title":"VisNumBench: Evaluating Number Sense of Multimodal Large Language Models","display_name":"VisNumBench: Evaluating Number Sense of Multimodal Large Language Models","publication_year":2025,"publication_date":"2025-10-19","ids":{"openalex":"https://openalex.org/W4414902532","doi":"https://doi.org/10.1109/iccv51701.2025.00365"},"language":"en","primary_location":{"id":"doi:10.1109/iccv51701.2025.00365","is_oa":false,"landing_page_url":"https://doi.org/10.1109/iccv51701.2025.00365","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 IEEE/CVF International Conference on Computer Vision (ICCV)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2503.14939","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5057294035","display_name":"Tengjin Weng","orcid":"https://orcid.org/0009-0006-9572-2576"},"institutions":[{"id":"https://openalex.org/I180726961","display_name":"Shenzhen University","ror":"https://ror.org/01vy4gh70","country_code":"CN","type":"education","lineage":["https://openalex.org/I180726961"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Tengjin Weng","raw_affiliation_strings":["College of Computer Science and Software Engineering, Shenzhen University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"College of Computer Science and Software Engineering, Shenzhen University","institution_ids":["https://openalex.org/I180726961"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100319491","display_name":"Jingyi Wang","orcid":"https://orcid.org/0000-0001-7113-7635"},"institutions":[{"id":"https://openalex.org/I3131625388","display_name":"University Town of Shenzhen","ror":"https://ror.org/05f5j6225","country_code":"CN","type":"education","lineage":["https://openalex.org/I3131625388"]},{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jingyi Wang","raw_affiliation_strings":["Shenzhen International Graduate School,Tsinghua University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Shenzhen International Graduate School,Tsinghua University","institution_ids":["https://openalex.org/I3131625388","https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101564877","display_name":"Wenhao Jiang","orcid":"https://orcid.org/0000-0001-6589-324X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wenhao Jiang","raw_affiliation_strings":["Guangdong Laboratory of Artificial Intelligence and Digital Economy (SZ)"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Guangdong Laboratory of Artificial Intelligence and Digital Economy (SZ)","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5108714406","display_name":"Ming Zhong","orcid":"https://orcid.org/0009-0005-8405-6830"},"institutions":[{"id":"https://openalex.org/I180726961","display_name":"Shenzhen University","ror":"https://ror.org/01vy4gh70","country_code":"CN","type":"education","lineage":["https://openalex.org/I180726961"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhong Ming","raw_affiliation_strings":["College of Computer Science and Software Engineering, Shenzhen University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"College of Computer Science and Software Engineering, Shenzhen University","institution_ids":["https://openalex.org/I180726961"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"3830","last_page":"3840"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9695000052452087,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9695000052452087,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13629","display_name":"Text Readability and Simplification","score":0.9589999914169312,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9459999799728394,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/number-sense","display_name":"Number sense","score":0.7261000275611877},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.6003999710083008},{"id":"https://openalex.org/keywords/range","display_name":"Range (aeronautics)","score":0.5442000031471252},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.48750001192092896},{"id":"https://openalex.org/keywords/visual-language","display_name":"Visual language","score":0.43230000138282776},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.4275999963283539}],"concepts":[{"id":"https://openalex.org/C173600914","wikidata":"https://www.wikidata.org/wiki/Q3663819","display_name":"Number sense","level":2,"score":0.7261000275611877},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6467999815940857},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.6003999710083008},{"id":"https://openalex.org/C204323151","wikidata":"https://www.wikidata.org/wiki/Q905424","display_name":"Range (aeronautics)","level":2,"score":0.5442000031471252},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.48750001192092896},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.48489999771118164},{"id":"https://openalex.org/C2780878386","wikidata":"https://www.wikidata.org/wiki/Q1659648","display_name":"Visual language","level":2,"score":0.43230000138282776},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.42910000681877136},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.4275999963283539},{"id":"https://openalex.org/C143141573","wikidata":"https://www.wikidata.org/wiki/Q7450971","display_name":"Sense (electronics)","level":2,"score":0.4214000105857849},{"id":"https://openalex.org/C66024118","wikidata":"https://www.wikidata.org/wiki/Q1122506","display_name":"Computational model","level":2,"score":0.34769999980926514},{"id":"https://openalex.org/C206345919","wikidata":"https://www.wikidata.org/wiki/Q20380951","display_name":"Resource (disambiguation)","level":2,"score":0.3260999917984009},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.30559998750686646},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.3009999990463257},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.25529998540878296},{"id":"https://openalex.org/C76969082","wikidata":"https://www.wikidata.org/wiki/Q486902","display_name":"Mathematical model","level":2,"score":0.25189998745918274}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1109/iccv51701.2025.00365","is_oa":false,"landing_page_url":"https://doi.org/10.1109/iccv51701.2025.00365","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 IEEE/CVF International Conference on Computer Vision (ICCV)","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2503.14939","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2503.14939","pdf_url":"https://arxiv.org/pdf/2503.14939","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2503.14939","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2503.14939","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2503.14939","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2503.14939","pdf_url":"https://arxiv.org/pdf/2503.14939","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Can":[0],"Multimodal":[1,113],"Large":[2],"Language":[3],"Models":[4],"(MLLMs)":[5],"develop":[6],"an":[7],"intuitive":[8],"number":[9,26,109,127,145,167],"sense":[10,27,128,146,168],"similar":[11],"to":[12,23,73],"humans?":[13],"Targeting":[14],"this":[15],"problem,":[16],"we":[17,82],"introduce":[18],"Visual":[19],"Number":[20],"Benchmark":[21],"(VisNumBench)":[22],"evaluate":[24],"the":[25,74,158],"abilities":[28,140],"of":[29,35,41,63],"MLLMs":[30,81,132],"across":[31],"a":[32,154],"wide":[33],"range":[34],"visual":[36,53,57,64],"numerical":[37,58,65],"tasks.":[38,67,111],"VisNumBench":[39,71,150],"consists":[40],"about":[42],"1,900":[43],"multiple-choice":[44],"question-answer":[45],"pairs":[46],"derived":[47],"from":[48],"both":[49],"synthetic":[50],"and":[51,60,90,99,116,137,171],"real-world":[52],"data,":[54],"covering":[55],"seven":[56],"attributes":[59],"four":[61],"types":[62],"estimation":[66],"Our":[68],"experiments":[69],"on":[70],"led":[72],"following":[75],"key":[76],"findings:":[77],"(i)":[78],"The":[79],"17":[80],"tested,":[83],"including":[84],"open-source":[85],"models":[86,96,115,120],"such":[87],"as":[88,92,94,153],"Qwen2.5-VL":[89],"InternVL2.5,":[91],"well":[93],"proprietary":[95],"like":[97],"GPT-4o":[98],"Gemini":[100],"2.0":[101],"Flash,":[102],"perform":[103],"significantly":[104],"below":[105],"human":[106],"levels":[107],"in":[108,126,144,164],"sense-related":[110],"(ii)":[112],"mathematical":[114],"multimodal":[117],"chain-of-thought":[118],"(CoT)":[119],"did":[121],"not":[122],"exhibit":[123],"significant":[124],"improvements":[125],"abilities.":[129,147,169],"(iii)":[130],"Stronger":[131],"with":[133],"larger":[134],"parameter":[135],"sizes":[136],"broader":[138],"general":[139],"demonstrate":[141],"modest":[142],"gains":[143],"We":[148],"believe":[149],"will":[151],"serve":[152],"valuable":[155],"resource":[156],"for":[157],"research":[159],"community,":[160],"encouraging":[161],"further":[162],"advancements":[163],"enhancing":[165],"MLLMs'":[166],"Code":[170],"dataset":[172],"are":[173],"available":[174],"at":[175],"https://wwwtttjjj.github.io/VisNumBench/.":[176]},"counts_by_year":[],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}