{"id":"https://openalex.org/W4415708032","doi":"https://doi.org/10.1109/icme59968.2025.11208956","title":"MLLM-DataEngine: Closing the Loop of Multimodal Instruction Tuning Data Generation","display_name":"MLLM-DataEngine: Closing the Loop of Multimodal Instruction Tuning Data Generation","publication_year":2025,"publication_date":"2025-06-30","ids":{"openalex":"https://openalex.org/W4415708032","doi":"https://doi.org/10.1109/icme59968.2025.11208956"},"language":null,"primary_location":{"id":"doi:10.1109/icme59968.2025.11208956","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icme59968.2025.11208956","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 IEEE International Conference on Multimedia and Expo (ICME)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2607.15299","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100575091","display_name":"Zhiyuan Zhao","orcid":null},"institutions":[{"id":"https://openalex.org/I4391012619","display_name":"Shanghai Artificial Intelligence Laboratory","ror":"https://ror.org/03wkvpx79","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4391012619"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhiyuan Zhao","raw_affiliation_strings":["Shanghai AI Laboratory,Shanghai,China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Shanghai AI Laboratory,Shanghai,China","institution_ids":["https://openalex.org/I4391012619"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101958711","display_name":"Bin Wang","orcid":"https://orcid.org/0000-0002-9607-1158"},"institutions":[{"id":"https://openalex.org/I4391012619","display_name":"Shanghai Artificial Intelligence Laboratory","ror":"https://ror.org/03wkvpx79","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4391012619"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Bin Wang","raw_affiliation_strings":["Shanghai AI Laboratory,Shanghai,China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Shanghai AI Laboratory,Shanghai,China","institution_ids":["https://openalex.org/I4391012619"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5104670548","display_name":"Linke Ouyang","orcid":null},"institutions":[{"id":"https://openalex.org/I4391012619","display_name":"Shanghai Artificial Intelligence Laboratory","ror":"https://ror.org/03wkvpx79","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4391012619"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Linke Ouyang","raw_affiliation_strings":["Shanghai AI Laboratory,Shanghai,China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Shanghai AI Laboratory,Shanghai,China","institution_ids":["https://openalex.org/I4391012619"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5074554822","display_name":"Yiqi Lin","orcid":"https://orcid.org/0000-0003-1914-7726"},"institutions":[{"id":"https://openalex.org/I4391012619","display_name":"Shanghai Artificial Intelligence Laboratory","ror":"https://ror.org/03wkvpx79","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4391012619"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yiqi Lin","raw_affiliation_strings":["Shanghai AI Laboratory,Shanghai,China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Shanghai AI Laboratory,Shanghai,China","institution_ids":["https://openalex.org/I4391012619"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100406359","display_name":"Pan Zhang","orcid":"https://orcid.org/0000-0003-2543-2275"},"institutions":[{"id":"https://openalex.org/I4391012619","display_name":"Shanghai Artificial Intelligence Laboratory","ror":"https://ror.org/03wkvpx79","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4391012619"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Pan Zhang","raw_affiliation_strings":["Shanghai AI Laboratory,Shanghai,China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Shanghai AI Laboratory,Shanghai,China","institution_ids":["https://openalex.org/I4391012619"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100303463","display_name":"Xiaoyi Dong","orcid":null},"institutions":[{"id":"https://openalex.org/I4391012619","display_name":"Shanghai Artificial Intelligence Laboratory","ror":"https://ror.org/03wkvpx79","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4391012619"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xiaoyi Dong","raw_affiliation_strings":["Shanghai AI Laboratory,Shanghai,China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Shanghai AI Laboratory,Shanghai,China","institution_ids":["https://openalex.org/I4391012619"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100365376","display_name":"Jiaqi Wang","orcid":"https://orcid.org/0000-0003-0210-0163"},"institutions":[{"id":"https://openalex.org/I4391012619","display_name":"Shanghai Artificial Intelligence Laboratory","ror":"https://ror.org/03wkvpx79","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4391012619"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jiaqi Wang","raw_affiliation_strings":["Shanghai AI Laboratory,Shanghai,China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Shanghai AI Laboratory,Shanghai,China","institution_ids":["https://openalex.org/I4391012619"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5101615091","display_name":"Conghui He","orcid":"https://orcid.org/0000-0001-8697-695X"},"institutions":[{"id":"https://openalex.org/I4391012619","display_name":"Shanghai Artificial Intelligence Laboratory","ror":"https://ror.org/03wkvpx79","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4391012619"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Conghui He","raw_affiliation_strings":["Shanghai AI Laboratory,Shanghai,China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Shanghai AI Laboratory,Shanghai,China","institution_ids":["https://openalex.org/I4391012619"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I4391012619"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"6"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.2134000062942505,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.2134000062942505,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13702","display_name":"Machine Learning in Healthcare","score":0.1331000030040741,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.08990000188350677,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/closing","display_name":"Closing (real estate)","score":0.6930999755859375},{"id":"https://openalex.org/keywords/benchmarking","display_name":"Benchmarking","score":0.6557000279426575},{"id":"https://openalex.org/keywords/data-collection","display_name":"Data collection","score":0.4424000084400177},{"id":"https://openalex.org/keywords/training-set","display_name":"Training set","score":0.3971000015735626},{"id":"https://openalex.org/keywords/data-modeling","display_name":"Data modeling","score":0.37220001220703125},{"id":"https://openalex.org/keywords/loop","display_name":"Loop (graph theory)","score":0.3452000021934509},{"id":"https://openalex.org/keywords/adaptive-sampling","display_name":"Adaptive sampling","score":0.3310000002384186},{"id":"https://openalex.org/keywords/class","display_name":"Class (philosophy)","score":0.3310000002384186}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7921000123023987},{"id":"https://openalex.org/C2778775528","wikidata":"https://www.wikidata.org/wiki/Q5135432","display_name":"Closing (real estate)","level":2,"score":0.6930999755859375},{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.6557000279426575},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5127999782562256},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4796000123023987},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.46860000491142273},{"id":"https://openalex.org/C133462117","wikidata":"https://www.wikidata.org/wiki/Q4929239","display_name":"Data collection","level":2,"score":0.4424000084400177},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.3971000015735626},{"id":"https://openalex.org/C67186912","wikidata":"https://www.wikidata.org/wiki/Q367664","display_name":"Data modeling","level":2,"score":0.37220001220703125},{"id":"https://openalex.org/C184670325","wikidata":"https://www.wikidata.org/wiki/Q512604","display_name":"Loop (graph theory)","level":2,"score":0.3452000021934509},{"id":"https://openalex.org/C2777212361","wikidata":"https://www.wikidata.org/wiki/Q5127848","display_name":"Class (philosophy)","level":2,"score":0.3310000002384186},{"id":"https://openalex.org/C2781395549","wikidata":"https://www.wikidata.org/wiki/Q4680762","display_name":"Adaptive sampling","level":3,"score":0.3310000002384186},{"id":"https://openalex.org/C2780626000","wikidata":"https://www.wikidata.org/wiki/Q5936775","display_name":"Human-in-the-loop","level":2,"score":0.32350000739097595},{"id":"https://openalex.org/C140779682","wikidata":"https://www.wikidata.org/wiki/Q210868","display_name":"Sampling (signal processing)","level":3,"score":0.3190999925136566},{"id":"https://openalex.org/C163985040","wikidata":"https://www.wikidata.org/wiki/Q1172399","display_name":"Data acquisition","level":2,"score":0.2904999852180481},{"id":"https://openalex.org/C9357733","wikidata":"https://www.wikidata.org/wiki/Q6878417","display_name":"Missing data","level":2,"score":0.2872999906539917},{"id":"https://openalex.org/C186886427","wikidata":"https://www.wikidata.org/wiki/Q5441213","display_name":"Feedback loop","level":2,"score":0.2858999967575073},{"id":"https://openalex.org/C2780735816","wikidata":"https://www.wikidata.org/wiki/Q28324931","display_name":"Incremental learning","level":2,"score":0.2849999964237213},{"id":"https://openalex.org/C3020493868","wikidata":"https://www.wikidata.org/wiki/Q55631277","display_name":"Real world data","level":2,"score":0.25380000472068787},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.25060001015663147}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1109/icme59968.2025.11208956","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icme59968.2025.11208956","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 IEEE International Conference on Multimedia and Expo (ICME)","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2607.15299","is_oa":true,"landing_page_url":"https://arxiv.org/abs/2607.15299","pdf_url":"https://arxiv.org/pdf/2607.15299","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2607.15299","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2607.15299","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2607.15299","is_oa":true,"landing_page_url":"https://arxiv.org/abs/2607.15299","pdf_url":"https://arxiv.org/pdf/2607.15299","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4415708032.pdf","grobid_xml":"https://content.openalex.org/works/W4415708032.grobid-xml"},"referenced_works_count":5,"referenced_works":["https://openalex.org/W2277195237","https://openalex.org/W4389520252","https://openalex.org/W4390874575","https://openalex.org/W4402727764","https://openalex.org/W4403081466"],"related_works":[],"abstract_inverted_index":{"In":[0],"this":[1],"paper,":[2],"we":[3,79,145],"propose":[4,80],"MLLM-DataEngine,":[5],"a":[6,38,154,167],"novel":[7],"closed-loop":[8],"system":[9],"that":[10],"bridges":[11],"data":[12,111,174],"generation,":[13],"model":[14,30,50,90,179],"training,":[15],"and":[16,47,71,97,121,136,156,178],"evaluation.":[17],"Within":[18],"each":[19],"loop":[20],"iteration,":[21,46],"the":[22,26,29,33,43,49,65,94,99,116,133,150,171],"MLLM-DataEngine":[23,67,147,164],"first":[24],"analyzes":[25],"weakness":[27,91,135],"of":[28,101],"based":[31,92],"on":[32,93],"evaluation":[34],"results,":[35],"then":[36],"generates":[37],"proper":[39],"incremental":[40,102],"dataset":[41,58],"for":[42,112,170],"next":[44],"training":[45],"enhances":[48],"capability":[51,114,152],"iteratively.":[52],"Compared":[53],"with":[54],"previous":[55],"instruction":[56],"fine-tuning":[57],"collection":[59],"methods":[60],"which":[61,86,128],"are":[62,124,180],"separate":[63],"from":[64],"benchmarking,":[66],"shows":[68],"better":[69],"targeting":[70],"can":[72,87],"improve":[73],"MLLMs\u2019s":[74],"capabilities":[75],"more":[76],"effectively.":[77],"Firstly,":[78],"an":[81],"Adaptive":[82],"Bad-case":[83],"Sampling":[84],"module,":[85],"effectively":[88],"analyze":[89],"benchmarking":[95],"results":[96],"adjust":[98],"generation":[100],"datasets":[103],"flexibly.":[104],"Secondly,":[105],"in":[106,153],"order":[107],"to":[108,126],"ensure":[109],"high-quality":[110,139],"specific":[113],"types,":[115],"most":[117],"representative":[118],"in-context":[119],"examples":[120],"abundant":[122],"information":[123],"provided":[125],"GPT-4,":[127],"helps":[129],"GPT-4":[130],"fully":[131],"comprehend":[132],"model\u2019s":[134],"further":[137],"guarantees":[138],"generated":[140],"data.":[141],"Through":[142],"extensive":[143],"experiments,":[144],"find":[146],"could":[148,165],"boost":[149],"MLLMs":[151,173],"targeted":[155],"automatic":[157],"manner":[158],"without":[159],"human":[160],"participants.":[161],"We":[162],"hope":[163],"be":[166],"general":[168],"solution":[169],"following":[172],"curation.":[175],"Code,":[176],"data,":[177],"available":[181],"at":[182],"https://github.com/opendatalab/MLLM-DataEngine.":[183]},"counts_by_year":[],"updated_date":"2026-07-23T08:03:31.855105","created_date":"2025-10-30T00:00:00"}