{"id":"https://openalex.org/W4402402086","doi":"https://doi.org/10.1109/tvcg.2024.3456159","title":"Advancing Multimodal Large Language Models in Chart Question Answering with Visualization-Referenced Instruction Tuning","display_name":"Advancing Multimodal Large Language Models in Chart Question Answering with Visualization-Referenced Instruction Tuning","publication_year":2024,"publication_date":"2024-09-10","ids":{"openalex":"https://openalex.org/W4402402086","doi":"https://doi.org/10.1109/tvcg.2024.3456159","pmid":"https://pubmed.ncbi.nlm.nih.gov/39255172"},"language":"en","primary_location":{"id":"doi:10.1109/tvcg.2024.3456159","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tvcg.2024.3456159","pdf_url":null,"source":{"id":"https://openalex.org/S84775595","display_name":"IEEE Transactions on Visualization and Computer Graphics","issn_l":"1077-2626","issn":["1077-2626","1941-0506","2160-9306"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Visualization and Computer Graphics","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5087805627","display_name":"Xingchen Zeng","orcid":"https://orcid.org/0000-0002-1967-0595"},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Xingchen Zeng","raw_affiliation_strings":["Hong Kong University of Science and Technology (Guangzhou), Guangzhou, China"],"affiliations":[{"raw_affiliation_string":"Hong Kong University of Science and Technology (Guangzhou), Guangzhou, China","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5104108869","display_name":"Haichuan Lin","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Haichuan Lin","raw_affiliation_strings":["Hong Kong University of Science and Technology (Guangzhou), Guangzhou, China"],"affiliations":[{"raw_affiliation_string":"Hong Kong University of Science and Technology (Guangzhou), Guangzhou, China","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5086643675","display_name":"Yilin Ye","orcid":"https://orcid.org/0000-0002-9451-9940"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yilin Ye","raw_affiliation_strings":["Hong Kong University of Science and Technology (Guangzhou), Guangzhou, China"],"affiliations":[{"raw_affiliation_string":"Hong Kong University of Science and Technology (Guangzhou), Guangzhou, China","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100660205","display_name":"Wei Zeng","orcid":"https://orcid.org/0000-0002-5600-8824"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wei Zeng","raw_affiliation_strings":["Hong Kong University of Science and Technology (Guangzhou), Guangzhou, China"],"affiliations":[{"raw_affiliation_string":"Hong Kong University of Science and Technology (Guangzhou), Guangzhou, China","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5087805627"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":6.1068,"has_fulltext":false,"cited_by_count":18,"citation_normalized_percentile":{"value":0.9685476,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":100},"biblio":{"volume":"31","issue":"1","first_page":"525","last_page":"535"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9898999929428101,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9898999929428101,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13083","display_name":"Advanced Text Analysis Techniques","score":0.9746999740600586,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9740999937057495,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.860090970993042},{"id":"https://openalex.org/keywords/visualization","display_name":"Visualization","score":0.7406505346298218},{"id":"https://openalex.org/keywords/question-answering","display_name":"Question answering","score":0.7341580390930176},{"id":"https://openalex.org/keywords/data-visualization","display_name":"Data visualization","score":0.5656291246414185},{"id":"https://openalex.org/keywords/chart","display_name":"Chart","score":0.5038198828697205},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.4584576189517975},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.45137113332748413},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4064696431159973},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.4041413366794586}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.860090970993042},{"id":"https://openalex.org/C36464697","wikidata":"https://www.wikidata.org/wiki/Q451553","display_name":"Visualization","level":2,"score":0.7406505346298218},{"id":"https://openalex.org/C44291984","wikidata":"https://www.wikidata.org/wiki/Q1074173","display_name":"Question answering","level":2,"score":0.7341580390930176},{"id":"https://openalex.org/C172367668","wikidata":"https://www.wikidata.org/wiki/Q6504956","display_name":"Data visualization","level":3,"score":0.5656291246414185},{"id":"https://openalex.org/C190812933","wikidata":"https://www.wikidata.org/wiki/Q28923","display_name":"Chart","level":2,"score":0.5038198828697205},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.4584576189517975},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.45137113332748413},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4064696431159973},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.4041413366794586},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1109/tvcg.2024.3456159","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tvcg.2024.3456159","pdf_url":null,"source":{"id":"https://openalex.org/S84775595","display_name":"IEEE Transactions on Visualization and Computer Graphics","issn_l":"1077-2626","issn":["1077-2626","1941-0506","2160-9306"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Visualization and Computer Graphics","raw_type":"journal-article"},{"id":"pmid:39255172","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/39255172","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE transactions on visualization and computer graphics","raw_type":null},{"id":"pmh:oai:repository.hkust.edu.hk:1783.1-147204","is_oa":false,"landing_page_url":"http://repository.hkust.edu.hk/ir/Record/1783.1-147204","pdf_url":null,"source":{"id":"https://openalex.org/S4306401796","display_name":"Rare & Special e-Zone (The Hong Kong University of Science and Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I200769079","host_organization_name":"Hong Kong University of Science and Technology","host_organization_lineage":["https://openalex.org/I200769079"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Quality Education","score":0.7400000095367432,"id":"https://metadata.un.org/sdg/4"}],"awards":[{"id":"https://openalex.org/G4023314675","display_name":null,"funder_award_id":"62172398","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":68,"referenced_works":["https://openalex.org/W1861492603","https://openalex.org/W1933349210","https://openalex.org/W2135415614","https://openalex.org/W2194775991","https://openalex.org/W2513780624","https://openalex.org/W2886887279","https://openalex.org/W2888029456","https://openalex.org/W2963351448","https://openalex.org/W2963427688","https://openalex.org/W3009518609","https://openalex.org/W3028907449","https://openalex.org/W3088290493","https://openalex.org/W3092587559","https://openalex.org/W3174906424","https://openalex.org/W3203626883","https://openalex.org/W3203907581","https://openalex.org/W4220657226","https://openalex.org/W4280596375","https://openalex.org/W4285255856","https://openalex.org/W4285292339","https://openalex.org/W4298326528","https://openalex.org/W4304480495","https://openalex.org/W4312233877","https://openalex.org/W4312443924","https://openalex.org/W4312695444","https://openalex.org/W4312846625","https://openalex.org/W4313524484","https://openalex.org/W4366550890","https://openalex.org/W4379374334","https://openalex.org/W4382132409","https://openalex.org/W4382240729","https://openalex.org/W4385565485","https://openalex.org/W4385570745","https://openalex.org/W4385570934","https://openalex.org/W4387934915","https://openalex.org/W4389519254","https://openalex.org/W4389523846","https://openalex.org/W4389524090","https://openalex.org/W4390872969","https://openalex.org/W4390886514","https://openalex.org/W4392538638","https://openalex.org/W4396832230","https://openalex.org/W4396851234","https://openalex.org/W4401042875","https://openalex.org/W4402683795","https://openalex.org/W4402727764","https://openalex.org/W4402753980","https://openalex.org/W6631190155","https://openalex.org/W6777615688","https://openalex.org/W6778883912","https://openalex.org/W6791353385","https://openalex.org/W6796581206","https://openalex.org/W6809646742","https://openalex.org/W6846178434","https://openalex.org/W6848678680","https://openalex.org/W6849177959","https://openalex.org/W6851147081","https://openalex.org/W6851592950","https://openalex.org/W6852874933","https://openalex.org/W6853469104","https://openalex.org/W6854262950","https://openalex.org/W6854336665","https://openalex.org/W6854866820","https://openalex.org/W6858878845","https://openalex.org/W6859483197","https://openalex.org/W6860198829","https://openalex.org/W6861859004","https://openalex.org/W6869270264"],"related_works":["https://openalex.org/W4288267738","https://openalex.org/W2964413124","https://openalex.org/W4388937922","https://openalex.org/W2013728941","https://openalex.org/W4225274103","https://openalex.org/W2579659702","https://openalex.org/W2154046714","https://openalex.org/W1965329638","https://openalex.org/W1574055964","https://openalex.org/W3113264705"],"abstract_inverted_index":{"Emerging":[0],"multimodal":[1],"large":[2],"language":[3],"models":[4,216],"(MLLMs)":[5],"exhibit":[6],"great":[7],"potential":[8],"for":[9,92,193,229],"chart":[10,100,171],"question":[11],"answering":[12],"(CQA).":[13],"Recent":[14],"efforts":[15],"primarily":[16],"focus":[17,55],"on":[18,40,56,217],"scaling":[19],"up":[20],"training":[21,84,121,208],"datasets":[22,45,143,235],"(i.e.,":[23],"charts,":[24],"data":[25,32,51,57,72,132,140,150,177],"tables,":[26],"and":[27,34,43,53,59,66,124,138,144,147,162,187,234],"question-answer":[28],"(QA)":[29],"pairs)":[30],"through":[31],"collection":[33,52],"synthesis.":[35],"However,":[36],"our":[37,203,210],"empirical":[38],"study":[39],"existing":[41,80,142],"MLLMs":[42,89],"CQA":[44,77,215],"reveals":[46],"notable":[47],"gaps.":[48],"First,":[49],"current":[50],"synthesis":[54],"volume":[58],"lack":[60],"consideration":[61],"of":[62,86,202,236],"fine-grained":[63,195],"visual":[64,163],"encodings":[65],"QA":[67,160],"tasks,":[68],"resulting":[69],"in":[70],"unbalanced":[71],"distribution":[73],"divergent":[74],"from":[75,141],"practical":[76,159],"scenarios.":[78],"Second,":[79],"work":[81],"follows":[82],"the":[83,87,96,109,120,149,168,175,184,200],"recipe":[85],"base":[88],"initially":[90],"designed":[91],"natural":[93],"images,":[94],"under-exploring":[95],"adaptation":[97,169,191],"to":[98,118,134,155,166,170,178],"unique":[99],"characteristics,":[101,172],"such":[102],"as":[103,226],"rich":[104],"text":[105],"elements.":[106],"To":[107],"fill":[108],"gap,":[110],"we":[111,128,173],"propose":[112,129],"a":[113,130,189,223,227],"visualization-referenced":[114],"instruction":[115],"tuning":[116],"approach":[117],"guide":[119],"dataset":[122,224],"enhancement":[123],"model":[125,211],"development.":[126],"Specifically,":[127],"novel":[131],"engine":[133],"effectively":[135],"filter":[136],"diverse":[137],"high-quality":[139],"subsequently":[145],"refine":[146],"augment":[148],"using":[151],"LLM-based":[152],"generation":[153],"techniques":[154],"better":[156],"align":[157],"with":[158,206],"tasks":[161],"encodings.":[164],"Then,":[165],"facilitate":[167],"utilize":[174],"enriched":[176],"train":[179],"an":[180],"MLLM":[181],"by":[182],"unfreezing":[183],"vision":[185],"encoder":[186],"incorporating":[188],"mixture-of-resolution":[190],"strategy":[192],"enhanced":[194],"recognition.":[196],"Experimental":[197],"results":[198],"validate":[199],"effectiveness":[201],"approach.":[204],"Even":[205],"fewer":[207],"examples,":[209],"consistently":[212],"outperforms":[213],"state-of-the-art":[214],"established":[218],"benchmarks.":[219],"We":[220],"also":[221],"contribute":[222],"split":[225],"benchmark":[228],"future":[230],"research.":[231],"Source":[232],"codes":[233],"this":[237],"paper":[238],"are":[239],"available":[240],"at":[241],"https://github.com/zengxingchen/ChartQA-MLLM.":[242]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":13},{"year":2024,"cited_by_count":2}],"updated_date":"2026-04-13T07:58:08.660418","created_date":"2025-10-10T00:00:00"}