{"id":"https://openalex.org/W4417283717","doi":"https://doi.org/10.1145/3748636.3762755","title":"CartoMapQA: A Fundamental Benchmark Dataset Evaluating Vision-Language Models on Cartographic Map Understanding","display_name":"CartoMapQA: A Fundamental Benchmark Dataset Evaluating Vision-Language Models on Cartographic Map Understanding","publication_year":2025,"publication_date":"2025-11-03","ids":{"openalex":"https://openalex.org/W4417283717","doi":"https://doi.org/10.1145/3748636.3762755"},"language":null,"primary_location":{"id":"doi:10.1145/3748636.3762755","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3748636.3762755","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 33rd ACM International Conference on Advances in Geographic Information Systems","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5035315327","display_name":"Huy Quang Ung","orcid":"https://orcid.org/0000-0001-9238-8601"},"institutions":[{"id":"https://openalex.org/I4210164495","display_name":"KDDI Research (Japan)","ror":"https://ror.org/05qsqt662","country_code":"JP","type":"company","lineage":["https://openalex.org/I4210164495"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Huy Quang Ung","raw_affiliation_strings":["KDDI Research, Inc., Fujimino, Japan"],"raw_orcid":"https://orcid.org/0000-0001-9238-8601","affiliations":[{"raw_affiliation_string":"KDDI Research, Inc., Fujimino, Japan","institution_ids":["https://openalex.org/I4210164495"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5015450941","display_name":"Guillaume Habault","orcid":"https://orcid.org/0000-0002-3364-5863"},"institutions":[{"id":"https://openalex.org/I4210164495","display_name":"KDDI Research (Japan)","ror":"https://ror.org/05qsqt662","country_code":"JP","type":"company","lineage":["https://openalex.org/I4210164495"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Guillaume Habault","raw_affiliation_strings":["KDDI Research, Inc., Fujimino, Japan"],"raw_orcid":"https://orcid.org/0000-0002-3364-5863","affiliations":[{"raw_affiliation_string":"KDDI Research, Inc., Fujimino, Japan","institution_ids":["https://openalex.org/I4210164495"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5053527032","display_name":"Yasutaka Nishimura","orcid":"https://orcid.org/0000-0003-4487-6285"},"institutions":[{"id":"https://openalex.org/I4210164495","display_name":"KDDI Research (Japan)","ror":"https://ror.org/05qsqt662","country_code":"JP","type":"company","lineage":["https://openalex.org/I4210164495"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Yasutaka Nishimura","raw_affiliation_strings":["KDDI Research, Inc., Fujimino, Japan"],"raw_orcid":"https://orcid.org/0000-0003-4487-6285","affiliations":[{"raw_affiliation_string":"KDDI Research, Inc., Fujimino, Japan","institution_ids":["https://openalex.org/I4210164495"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100619561","display_name":"Hao Niu","orcid":"https://orcid.org/0000-0002-5623-9470"},"institutions":[{"id":"https://openalex.org/I4210164495","display_name":"KDDI Research (Japan)","ror":"https://ror.org/05qsqt662","country_code":"JP","type":"company","lineage":["https://openalex.org/I4210164495"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Hao Niu","raw_affiliation_strings":["KDDI Research, Inc., Fujimino, Japan"],"raw_orcid":"https://orcid.org/0000-0002-5623-9470","affiliations":[{"raw_affiliation_string":"KDDI Research, Inc., Fujimino, Japan","institution_ids":["https://openalex.org/I4210164495"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5014746910","display_name":"Roberto Legaspi","orcid":"https://orcid.org/0000-0001-8909-635X"},"institutions":[{"id":"https://openalex.org/I4210164495","display_name":"KDDI Research (Japan)","ror":"https://ror.org/05qsqt662","country_code":"JP","type":"company","lineage":["https://openalex.org/I4210164495"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Roberto Legaspi","raw_affiliation_strings":["KDDI Research, Inc., Fujimino, Japan"],"raw_orcid":"https://orcid.org/0000-0001-8909-635X","affiliations":[{"raw_affiliation_string":"KDDI Research, Inc., Fujimino, Japan","institution_ids":["https://openalex.org/I4210164495"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5050050520","display_name":"Tomoki Oya","orcid":"https://orcid.org/0009-0005-4927-076X"},"institutions":[{"id":"https://openalex.org/I4210164495","display_name":"KDDI Research (Japan)","ror":"https://ror.org/05qsqt662","country_code":"JP","type":"company","lineage":["https://openalex.org/I4210164495"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Tomoki Oya","raw_affiliation_strings":["KDDI Research, Inc., Fujimino, Japan"],"raw_orcid":"https://orcid.org/0009-0005-4927-076X","affiliations":[{"raw_affiliation_string":"KDDI Research, Inc., Fujimino, Japan","institution_ids":["https://openalex.org/I4210164495"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5058060139","display_name":"Ryoichi Kojima","orcid":"https://orcid.org/0009-0009-3128-7781"},"institutions":[{"id":"https://openalex.org/I4210164495","display_name":"KDDI Research (Japan)","ror":"https://ror.org/05qsqt662","country_code":"JP","type":"company","lineage":["https://openalex.org/I4210164495"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Ryoichi Kojima","raw_affiliation_strings":["KDDI Research, Inc., Fujimino, Japan"],"raw_orcid":"https://orcid.org/0009-0009-3128-7781","affiliations":[{"raw_affiliation_string":"KDDI Research, Inc., Fujimino, Japan","institution_ids":["https://openalex.org/I4210164495"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5084224574","display_name":"Masato Taya","orcid":"https://orcid.org/0009-0006-4911-4289"},"institutions":[{"id":"https://openalex.org/I4210164495","display_name":"KDDI Research (Japan)","ror":"https://ror.org/05qsqt662","country_code":"JP","type":"company","lineage":["https://openalex.org/I4210164495"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Masato Taya","raw_affiliation_strings":["KDDI Research, Inc., Fujimino, Japan"],"raw_orcid":"https://orcid.org/0009-0006-4911-4289","affiliations":[{"raw_affiliation_string":"KDDI Research, Inc., Fujimino, Japan","institution_ids":["https://openalex.org/I4210164495"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5033829205","display_name":"Chihiro Ono","orcid":"https://orcid.org/0000-0002-6410-1359"},"institutions":[{"id":"https://openalex.org/I4210164495","display_name":"KDDI Research (Japan)","ror":"https://ror.org/05qsqt662","country_code":"JP","type":"company","lineage":["https://openalex.org/I4210164495"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Chihiro Ono","raw_affiliation_strings":["KDDI Research, Inc., Fujimino, Japan"],"raw_orcid":"https://orcid.org/0000-0002-6410-1359","affiliations":[{"raw_affiliation_string":"KDDI Research, Inc., Fujimino, Japan","institution_ids":["https://openalex.org/I4210164495"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5083082197","display_name":"Atsunori Minamikawa","orcid":"https://orcid.org/0009-0009-8856-7813"},"institutions":[{"id":"https://openalex.org/I4210164495","display_name":"KDDI Research (Japan)","ror":"https://ror.org/05qsqt662","country_code":"JP","type":"company","lineage":["https://openalex.org/I4210164495"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Atsunori Minamikawa","raw_affiliation_strings":["KDDI Research, Inc., Fujimino, Japan"],"raw_orcid":"https://orcid.org/0009-0009-8856-7813","affiliations":[{"raw_affiliation_string":"KDDI Research, Inc., Fujimino, Japan","institution_ids":["https://openalex.org/I4210164495"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5101823667","display_name":"Yan Liu","orcid":"https://orcid.org/0000-0002-7055-9518"},"institutions":[{"id":"https://openalex.org/I1174212","display_name":"University of Southern California","ror":"https://ror.org/03taz7m60","country_code":"US","type":"education","lineage":["https://openalex.org/I1174212"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yan Liu","raw_affiliation_strings":["University of Southern California, Los Angeles, California, USA"],"raw_orcid":"https://orcid.org/0000-0002-7055-9518","affiliations":[{"raw_affiliation_string":"University of Southern California, Los Angeles, California, USA","institution_ids":["https://openalex.org/I1174212"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":11,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.35759066,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"440","last_page":"453"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9067000150680542,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9067000150680542,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10627","display_name":"Advanced Image and Video Retrieval Techniques","score":0.029999999329447746,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.011300000362098217,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/geospatial-analysis","display_name":"Geospatial analysis","score":0.7703999876976013},{"id":"https://openalex.org/keywords/cartographic-generalization","display_name":"Cartographic generalization","score":0.6388000249862671},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.6129999756813049},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.5942999720573425},{"id":"https://openalex.org/keywords/interpretation","display_name":"Interpretation (philosophy)","score":0.4984000027179718},{"id":"https://openalex.org/keywords/scale","display_name":"Scale (ratio)","score":0.4726000130176544},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.44670000672340393},{"id":"https://openalex.org/keywords/visualization","display_name":"Visualization","score":0.3741999864578247}],"concepts":[{"id":"https://openalex.org/C9770341","wikidata":"https://www.wikidata.org/wiki/Q1938983","display_name":"Geospatial analysis","level":2,"score":0.7703999876976013},{"id":"https://openalex.org/C196031653","wikidata":"https://www.wikidata.org/wiki/Q1501867","display_name":"Cartographic generalization","level":3,"score":0.6388000249862671},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.614300012588501},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.6129999756813049},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.5942999720573425},{"id":"https://openalex.org/C527412718","wikidata":"https://www.wikidata.org/wiki/Q855395","display_name":"Interpretation (philosophy)","level":2,"score":0.4984000027179718},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.4726000130176544},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.44670000672340393},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.4277999997138977},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.38499999046325684},{"id":"https://openalex.org/C36464697","wikidata":"https://www.wikidata.org/wiki/Q451553","display_name":"Visualization","level":2,"score":0.3741999864578247},{"id":"https://openalex.org/C134400042","wikidata":"https://www.wikidata.org/wiki/Q2372244","display_name":"Symbol (formal)","level":2,"score":0.3725999891757965},{"id":"https://openalex.org/C58640448","wikidata":"https://www.wikidata.org/wiki/Q42515","display_name":"Cartography","level":1,"score":0.37059998512268066},{"id":"https://openalex.org/C159620131","wikidata":"https://www.wikidata.org/wiki/Q1938983","display_name":"Spatial analysis","level":2,"score":0.3587999939918518},{"id":"https://openalex.org/C41856607","wikidata":"https://www.wikidata.org/wiki/Q483130","display_name":"Geographic information system","level":2,"score":0.35519999265670776},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.34850001335144043},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.3465999960899353},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.34619998931884766},{"id":"https://openalex.org/C9652623","wikidata":"https://www.wikidata.org/wiki/Q190109","display_name":"Field (mathematics)","level":2,"score":0.3059000074863434},{"id":"https://openalex.org/C67186912","wikidata":"https://www.wikidata.org/wiki/Q367664","display_name":"Data modeling","level":2,"score":0.2888999879360199},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.28130000829696655},{"id":"https://openalex.org/C93692415","wikidata":"https://www.wikidata.org/wiki/Q1502030","display_name":"Thematic map","level":2,"score":0.2734000086784363},{"id":"https://openalex.org/C180198813","wikidata":"https://www.wikidata.org/wiki/Q121182","display_name":"Information system","level":2,"score":0.258899986743927},{"id":"https://openalex.org/C2780861071","wikidata":"https://www.wikidata.org/wiki/Q1062934","display_name":"Character (mathematics)","level":2,"score":0.2572999894618988}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3748636.3762755","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3748636.3762755","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 33rd ACM International Conference on Advances in Geographic Information Systems","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":10,"referenced_works":["https://openalex.org/W3012111773","https://openalex.org/W4385571157","https://openalex.org/W4388834509","https://openalex.org/W4390100400","https://openalex.org/W4392384758","https://openalex.org/W4396735245","https://openalex.org/W4402716477","https://openalex.org/W4402727764","https://openalex.org/W4402727885","https://openalex.org/W4405595839"],"related_works":[],"abstract_inverted_index":{"The":[0,48],"rise":[1],"of":[2,42,56,95,144],"Large":[3],"Visual-Language":[4],"Models":[5],"(LVLMs)":[6],"has":[7],"unlocked":[8],"new":[9],"possibilities":[10],"for":[11,132,148],"seamlessly":[12],"integrating":[13],"visual":[14],"and":[15,67,77,90,98,114,155,164,170],"textual":[16],"information.":[17],"However,":[18],"their":[19],"ability":[20],"to":[21,38,117,175],"interpret":[22],"cartographic":[23,43,58],"maps":[24,44],"remains":[25],"largely":[26],"unexplored.":[27],"In":[28],"this":[29],"paper,":[30],"we":[31],"introduce":[32],"CartoMapQA,":[33],"a":[34,57,60,68,129],"benchmark":[35],"specifically":[36],"designed":[37],"evaluate":[39],"LVLMs'":[40],"understanding":[41],"through":[45],"question-answering":[46],"tasks.":[47],"dataset":[49],"includes":[50],"over":[51],"2000":[52],"samples,":[53],"each":[54],"composed":[55],"map,":[59],"question":[61],"(with":[62],"open-ended":[63],"or":[64],"multiple-choice":[65],"answers),":[66],"ground-truth":[69],"answer.":[70],"These":[71],"tasks":[72],"span":[73],"key":[74],"low-,":[75],"mid-":[76],"high-level":[78],"map":[79,157],"interpretation":[80],"skills,":[81],"including":[82],"symbol":[83],"recognition,":[84],"embedded":[85],"information":[86],"extraction,":[87],"scale":[88],"interpretation,":[89],"route-based":[91],"reasoning.":[92],"Our":[93,167],"evaluation":[94],"both":[96],"open-source":[97],"proprietary":[99],"LVLMs":[100],"reveals":[101],"persistent":[102],"challenges:":[103],"models":[104,145],"frequently":[105],"struggle":[106],"with":[107],"map-specific":[108],"semantics,":[109],"exhibit":[110],"limited":[111],"geospatial":[112],"reasoning,":[113],"are":[115,172],"prone":[116],"Optical":[118],"Character":[119],"Recognition":[120],"(OCR)-related":[121],"errors.":[122],"By":[123],"isolating":[124],"these":[125],"weaknesses,":[126],"CartoMapQA":[127],"offers":[128],"valuable":[130],"tool":[131],"guiding":[133],"future":[134],"improvements":[135],"in":[136],"LVLM":[137],"architectures.":[138],"Ultimately,":[139],"it":[140],"supports":[141],"the":[142,176],"development":[143],"better":[146],"equipped":[147],"real-world":[149],"applications":[150],"that":[151],"depend":[152],"on":[153],"robust":[154],"reliable":[156],"understanding,":[158],"such":[159],"as":[160],"navigation,":[161],"geographic":[162],"search,":[163],"urban":[165],"planning.":[166],"source":[168],"code":[169],"data":[171],"openly":[173],"available":[174],"research":[177],"community":[178],"at:":[179],"https://github.com/ungquanghuy-kddi/CartoMapQA.git":[180]},"counts_by_year":[],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-12-12T00:00:00"}