{"id":"https://openalex.org/W4393259344","doi":"https://doi.org/10.1145/3654674","title":"TOMGPT: Reliable Text-Only Training Approach for Cost-Effective Multi-modal Large Language Model","display_name":"TOMGPT: Reliable Text-Only Training Approach for Cost-Effective Multi-modal Large Language Model","publication_year":2024,"publication_date":"2024-03-28","ids":{"openalex":"https://openalex.org/W4393259344","doi":"https://doi.org/10.1145/3654674"},"language":"en","primary_location":{"id":"doi:10.1145/3654674","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3654674","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3654674","source":{"id":"https://openalex.org/S41523882","display_name":"ACM Transactions on Knowledge Discovery from Data","issn_l":"1556-4681","issn":["1556-4681","1556-472X"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Knowledge Discovery from Data","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3654674","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5065611010","display_name":"Yunkai Chen","orcid":"https://orcid.org/0000-0002-7197-246X"},"institutions":[{"id":"https://openalex.org/I126520041","display_name":"University of Science and Technology of China","ror":"https://ror.org/04c4dkn09","country_code":"CN","type":"education","lineage":["https://openalex.org/I126520041","https://openalex.org/I19820366"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Yunkai Chen","raw_affiliation_strings":["University of Science and Technology of China, Hefei, China","University of Science and Technology of China, Hefei City, China"],"affiliations":[{"raw_affiliation_string":"University of Science and Technology of China, Hefei, China","institution_ids":["https://openalex.org/I126520041"]},{"raw_affiliation_string":"University of Science and Technology of China, Hefei City, China","institution_ids":["https://openalex.org/I126520041"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5083340374","display_name":"Qimeng Wang","orcid":"https://orcid.org/0000-0002-9715-836X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Qimeng Wang","raw_affiliation_strings":["Xiaohongshu Inc., Beijing, China"],"affiliations":[{"raw_affiliation_string":"Xiaohongshu Inc., Beijing, China","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101934104","display_name":"Shiwei Wu","orcid":"https://orcid.org/0000-0002-3206-6827"},"institutions":[{"id":"https://openalex.org/I126520041","display_name":"University of Science and Technology of China","ror":"https://ror.org/04c4dkn09","country_code":"CN","type":"education","lineage":["https://openalex.org/I126520041","https://openalex.org/I19820366"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Shiwei Wu","raw_affiliation_strings":["University of Science and Technology of China, Hefei, China","University of Science and Technology of China, Hefei City, China"],"affiliations":[{"raw_affiliation_string":"University of Science and Technology of China, Hefei, China","institution_ids":["https://openalex.org/I126520041"]},{"raw_affiliation_string":"University of Science and Technology of China, Hefei City, China","institution_ids":["https://openalex.org/I126520041"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102374746","display_name":"Yan Gao","orcid":"https://orcid.org/0009-0004-5960-1684"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yan Gao","raw_affiliation_strings":["Xiaohongshu Inc., Beijing, China"],"affiliations":[{"raw_affiliation_string":"Xiaohongshu Inc., Beijing, China","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5025292786","display_name":"Tong Xu","orcid":"https://orcid.org/0000-0003-4246-5386"},"institutions":[{"id":"https://openalex.org/I126520041","display_name":"University of Science and Technology of China","ror":"https://ror.org/04c4dkn09","country_code":"CN","type":"education","lineage":["https://openalex.org/I126520041","https://openalex.org/I19820366"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Tong Xu","raw_affiliation_strings":["University of Science and Technology of China, Hefei, China","University of Science and Technology of China, Hefei City, China"],"affiliations":[{"raw_affiliation_string":"University of Science and Technology of China, Hefei, China","institution_ids":["https://openalex.org/I126520041"]},{"raw_affiliation_string":"University of Science and Technology of China, Hefei City, China","institution_ids":["https://openalex.org/I126520041"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5107062676","display_name":"Yao Hu","orcid":"https://orcid.org/0009-0006-1274-7111"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yao Hu","raw_affiliation_strings":["Xiaohongshu Inc., Beijing, China"],"affiliations":[{"raw_affiliation_string":"Xiaohongshu Inc., Beijing, China","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5065611010"],"corresponding_institution_ids":["https://openalex.org/I126520041"],"apc_list":null,"apc_paid":null,"fwci":4.3796,"has_fulltext":true,"cited_by_count":13,"citation_normalized_percentile":{"value":0.9482757,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":96,"max":99},"biblio":{"volume":"18","issue":"7","first_page":"1","last_page":"19"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6824455857276917},{"id":"https://openalex.org/keywords/modal","display_name":"Modal","score":0.6496949195861816},{"id":"https://openalex.org/keywords/training","display_name":"Training (meteorology)","score":0.5732710957527161},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.4498419761657715},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4435766637325287},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.33196431398391724},{"id":"https://openalex.org/keywords/geography","display_name":"Geography","score":0.06850144267082214},{"id":"https://openalex.org/keywords/chemistry","display_name":"Chemistry","score":0.06634914875030518}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6824455857276917},{"id":"https://openalex.org/C71139939","wikidata":"https://www.wikidata.org/wiki/Q910194","display_name":"Modal","level":2,"score":0.6496949195861816},{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.5732710957527161},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.4498419761657715},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4435766637325287},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.33196431398391724},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.06850144267082214},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.06634914875030518},{"id":"https://openalex.org/C188027245","wikidata":"https://www.wikidata.org/wiki/Q750446","display_name":"Polymer chemistry","level":1,"score":0.0},{"id":"https://openalex.org/C153294291","wikidata":"https://www.wikidata.org/wiki/Q25261","display_name":"Meteorology","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3654674","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3654674","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3654674","source":{"id":"https://openalex.org/S41523882","display_name":"ACM Transactions on Knowledge Discovery from Data","issn_l":"1556-4681","issn":["1556-4681","1556-472X"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Knowledge Discovery from Data","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1145/3654674","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3654674","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3654674","source":{"id":"https://openalex.org/S41523882","display_name":"ACM Transactions on Knowledge Discovery from Data","issn_l":"1556-4681","issn":["1556-4681","1556-472X"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Knowledge Discovery from Data","raw_type":"journal-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","score":0.6600000262260437,"display_name":"Quality Education"}],"awards":[{"id":"https://openalex.org/G1198996116","display_name":null,"funder_award_id":"62222213, U22B2059, 62072423","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G3317480652","display_name":null,"funder_award_id":"Science","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G3604000827","display_name":null,"funder_award_id":"62222213","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G3841266055","display_name":null,"funder_award_id":"6222221","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G4442529403","display_name":null,"funder_award_id":"U22B2059","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G5994120800","display_name":null,"funder_award_id":"Natural","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G7295420491","display_name":null,"funder_award_id":"62072423","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4393259344.pdf","grobid_xml":"https://content.openalex.org/works/W4393259344.grobid-xml"},"referenced_works_count":37,"referenced_works":["https://openalex.org/W1593271688","https://openalex.org/W2602856279","https://openalex.org/W3034999214","https://openalex.org/W3198377975","https://openalex.org/W3205021045","https://openalex.org/W4229005866","https://openalex.org/W4292779060","https://openalex.org/W4292828275","https://openalex.org/W4307106676","https://openalex.org/W4308760226","https://openalex.org/W4311642023","https://openalex.org/W4312310776","https://openalex.org/W4312933868","https://openalex.org/W4312938727","https://openalex.org/W4313136445","https://openalex.org/W4322718191","https://openalex.org/W4362515116","https://openalex.org/W4367367040","https://openalex.org/W4367628410","https://openalex.org/W4375869762","https://openalex.org/W4382142077","https://openalex.org/W4383473994","https://openalex.org/W4384918448","https://openalex.org/W4385570984","https://openalex.org/W4385571029","https://openalex.org/W4385682171","https://openalex.org/W4386184788","https://openalex.org/W4390889743","https://openalex.org/W4391136507","https://openalex.org/W4393065402","https://openalex.org/W6778883912","https://openalex.org/W6803567076","https://openalex.org/W6811340617","https://openalex.org/W6851813333","https://openalex.org/W6854692045","https://openalex.org/W6855592305","https://openalex.org/W6912494966"],"related_works":["https://openalex.org/W2961085424","https://openalex.org/W4306674287","https://openalex.org/W3046775127","https://openalex.org/W3107602296","https://openalex.org/W3170094116","https://openalex.org/W4386462264","https://openalex.org/W4364306694","https://openalex.org/W4312192474","https://openalex.org/W4283697347","https://openalex.org/W4210805261"],"abstract_inverted_index":{"Multi-modal":[0,68],"large":[1,196],"language":[2],"models":[3,192],"(MLLMs),":[4],"such":[5],"as":[6,15,17,150],"GPT-4,":[7],"exhibit":[8],"great":[9],"comprehension":[10,119],"capabilities":[11,120,213],"on":[12,20,75,167,194],"human":[13],"instruction,":[14],"well":[16],"zero-shot":[18],"ability":[19],"new":[21],"downstream":[22],"multi-modal":[23,107],"tasks.":[24],"To":[25],"integrate":[26],"the":[27,105,115,153],"different":[28],"modalities":[29],"within":[30],"a":[31,70,96,195],"unified":[32],"embedding":[33],"space,":[34],"previous":[35,133],"MLLMs":[36],"attempted":[37],"to":[38,102,110,190],"conduct":[39],"visual":[40,118],"instruction":[41],"tuning":[42],"with":[43,80,85,117,142],"massive":[44],"and":[45,57,93,158,170,176,211],"high-quality":[46],"image-text":[47,128,199],"pair":[48,200],"data,":[49,149],"which":[50,172],"requires":[51],"substantial":[52],"costs":[53],"in":[54,121],"data":[55,79,130],"collection":[56],"training":[58,67,98,129],"resources.":[59,83],"In":[60],"this":[61],"article,":[62],"we":[63,135,151],"propose":[64],"TOMGPT":[65,138,185],"(Text-Only":[66],"GPT),":[69],"cost-effective":[71],"MLLM":[72,179],"tuned":[73],"solely":[74],"easily":[76],"accessible":[77],"text":[78,148],"much":[81],"fewer":[82,143],"Along":[84],"pre-trained":[86,159],"visual-linguistic":[87],"coupled":[88],"modality":[89],"space":[90,109],"(e.g.,":[91],"CLIP":[92],"ALIGN":[94],"model),":[95],"text-only":[97],"strategy":[99],"is":[100,165],"devised":[101],"further":[103],"project":[104],"aligned":[106],"latent":[108],"that":[111,137,184],"of":[112,126,198],"LLM,":[113],"endowing":[114],"LLM":[116,157],"an":[122],"efficient":[123],"manner.":[124],"Instead":[125],"enormous":[127],"required":[131],"by":[132],"MLLMs,":[134],"find":[136],"can":[139],"be":[140],"well-tuned":[141],"yet":[144],"diverse":[145,215],"GPT-generated":[146],"free-form":[147],"establish":[152],"semantic":[154],"connection":[155],"between":[156],"vision-language":[160],"model.":[161],"A":[162],"quantitative":[163],"evaluation":[164],"conducted":[166],"both":[168],"MME":[169],"LVLM,":[171],"are":[173,204],"recently":[174],"released":[175],"extensively":[177],"utilized":[178],"benchmarks.":[180],"The":[181],"experiments":[182],"reveal":[183],"achieved":[186],"reliable":[187],"performance":[188],"compared":[189],"numerous":[191],"trained":[193],"amount":[197],"data.":[201],"Case":[202],"studies":[203],"also":[205],"presented,":[206],"demonstrating":[207],"TOMGPT\u2019s":[208],"broad":[209],"understanding":[210],"dialogue":[212],"across":[214],"image":[216],"categories.":[217]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":7},{"year":2024,"cited_by_count":5}],"updated_date":"2026-04-16T08:26:57.006410","created_date":"2025-10-10T00:00:00"}