{"id":"https://openalex.org/W4403792205","doi":"https://doi.org/10.1145/3664647.3681294","title":"mPLUG-PaperOwl: Scientific Diagram Analysis with the Multimodal Large Language Model","display_name":"mPLUG-PaperOwl: Scientific Diagram Analysis with the Multimodal Large Language Model","publication_year":2024,"publication_date":"2024-10-26","ids":{"openalex":"https://openalex.org/W4403792205","doi":"https://doi.org/10.1145/3664647.3681294"},"language":"en","primary_location":{"id":"doi:10.1145/3664647.3681294","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3664647.3681294","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 32nd ACM International Conference on Multimedia","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5019498452","display_name":"Anwen Hu","orcid":"https://orcid.org/0000-0001-8839-4996"},"institutions":[{"id":"https://openalex.org/I45928872","display_name":"Alibaba Group (China)","ror":"https://ror.org/00k642b80","country_code":"CN","type":"company","lineage":["https://openalex.org/I45928872"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Anwen Hu","raw_affiliation_strings":["Alibaba Group, Chaoyang Qu, Beijing Shi, China"],"raw_orcid":"https://orcid.org/0000-0001-8839-4996","affiliations":[{"raw_affiliation_string":"Alibaba Group, Chaoyang Qu, Beijing Shi, China","institution_ids":["https://openalex.org/I45928872"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5019915855","display_name":"Yaya Shi","orcid":"https://orcid.org/0000-0003-0465-6712"},"institutions":[{"id":"https://openalex.org/I126520041","display_name":"University of Science and Technology of China","ror":"https://ror.org/04c4dkn09","country_code":"CN","type":"education","lineage":["https://openalex.org/I126520041","https://openalex.org/I19820366"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yaya Shi","raw_affiliation_strings":["University of Science and Technology of China, Hefei, Anhui, China"],"raw_orcid":"https://orcid.org/0000-0003-0465-6712","affiliations":[{"raw_affiliation_string":"University of Science and Technology of China, Hefei, Anhui, China","institution_ids":["https://openalex.org/I126520041"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5066544410","display_name":"Haiyang Xu","orcid":"https://orcid.org/0000-0001-9442-5912"},"institutions":[{"id":"https://openalex.org/I45928872","display_name":"Alibaba Group (China)","ror":"https://ror.org/00k642b80","country_code":"CN","type":"company","lineage":["https://openalex.org/I45928872"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Haiyang Xu","raw_affiliation_strings":["Alibaba Group, HangZhou, Zhejiang, China"],"raw_orcid":"https://orcid.org/0000-0001-9442-5912","affiliations":[{"raw_affiliation_string":"Alibaba Group, HangZhou, Zhejiang, China","institution_ids":["https://openalex.org/I45928872"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5091465907","display_name":"Jiabo Ye","orcid":"https://orcid.org/0009-0009-5451-8984"},"institutions":[{"id":"https://openalex.org/I66867065","display_name":"East China Normal University","ror":"https://ror.org/02n96ep67","country_code":"CN","type":"education","lineage":["https://openalex.org/I66867065"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jiabo Ye","raw_affiliation_strings":["East China Normal University, Putuo Qu, Shanghai, China"],"raw_orcid":"https://orcid.org/0009-0009-5451-8984","affiliations":[{"raw_affiliation_string":"East China Normal University, Putuo Qu, Shanghai, China","institution_ids":["https://openalex.org/I66867065"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5005965903","display_name":"Qinghao Ye","orcid":"https://orcid.org/0000-0002-7977-5540"},"institutions":[{"id":"https://openalex.org/I45928872","display_name":"Alibaba Group (China)","ror":"https://ror.org/00k642b80","country_code":"CN","type":"company","lineage":["https://openalex.org/I45928872"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Qinghao Ye","raw_affiliation_strings":["Alibaba Group, HangZhou, Zhejiang, China"],"raw_orcid":"https://orcid.org/0000-0002-7977-5540","affiliations":[{"raw_affiliation_string":"Alibaba Group, HangZhou, Zhejiang, China","institution_ids":["https://openalex.org/I45928872"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100452477","display_name":"Ming Yan","orcid":"https://orcid.org/0000-0003-4959-8878"},"institutions":[{"id":"https://openalex.org/I45928872","display_name":"Alibaba Group (China)","ror":"https://ror.org/00k642b80","country_code":"CN","type":"company","lineage":["https://openalex.org/I45928872"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Ming Yan","raw_affiliation_strings":["Alibaba Group, HangZhou, Zhejiang, China"],"raw_orcid":"https://orcid.org/0000-0003-4959-8878","affiliations":[{"raw_affiliation_string":"Alibaba Group, HangZhou, Zhejiang, China","institution_ids":["https://openalex.org/I45928872"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100734065","display_name":"Chenliang Li","orcid":"https://orcid.org/0000-0001-9077-3928"},"institutions":[{"id":"https://openalex.org/I45928872","display_name":"Alibaba Group (China)","ror":"https://ror.org/00k642b80","country_code":"CN","type":"company","lineage":["https://openalex.org/I45928872"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Chenliang Li","raw_affiliation_strings":["Alibaba Group, HangZhou, Zhejiang, China"],"raw_orcid":"https://orcid.org/0000-0001-9077-3928","affiliations":[{"raw_affiliation_string":"Alibaba Group, HangZhou, Zhejiang, China","institution_ids":["https://openalex.org/I45928872"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5111800002","display_name":"Qi Qian","orcid":null},"institutions":[{"id":"https://openalex.org/I4210095624","display_name":"Alibaba Group (United States)","ror":"https://ror.org/00rn0m335","country_code":"US","type":"company","lineage":["https://openalex.org/I4210095624","https://openalex.org/I45928872"]},{"id":"https://openalex.org/I4210108985","display_name":"Bellevue Hospital Center","ror":"https://ror.org/01ky34z31","country_code":"US","type":"healthcare","lineage":["https://openalex.org/I1283621791","https://openalex.org/I4210086933","https://openalex.org/I4210108985"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Qi Qian","raw_affiliation_strings":["Alibaba Group, Bellevue, WA, USA"],"raw_orcid":"https://orcid.org/0009-0007-8661-1169","affiliations":[{"raw_affiliation_string":"Alibaba Group, Bellevue, WA, USA","institution_ids":["https://openalex.org/I4210095624","https://openalex.org/I4210108985"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100329266","display_name":"Ji Zhang","orcid":"https://orcid.org/0000-0002-3835-7975"},"institutions":[{"id":"https://openalex.org/I45928872","display_name":"Alibaba Group (China)","ror":"https://ror.org/00k642b80","country_code":"CN","type":"company","lineage":["https://openalex.org/I45928872"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Ji Zhang","raw_affiliation_strings":["Alibaba Group, HangZhou, Zhejiang, China"],"raw_orcid":"https://orcid.org/0000-0002-3835-7975","affiliations":[{"raw_affiliation_string":"Alibaba Group, HangZhou, Zhejiang, China","institution_ids":["https://openalex.org/I45928872"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5101488344","display_name":"Fei Huang","orcid":"https://orcid.org/0000-0002-3709-5053"},"institutions":[{"id":"https://openalex.org/I4210095624","display_name":"Alibaba Group (United States)","ror":"https://ror.org/00rn0m335","country_code":"US","type":"company","lineage":["https://openalex.org/I4210095624","https://openalex.org/I45928872"]},{"id":"https://openalex.org/I4210108985","display_name":"Bellevue Hospital Center","ror":"https://ror.org/01ky34z31","country_code":"US","type":"healthcare","lineage":["https://openalex.org/I1283621791","https://openalex.org/I4210086933","https://openalex.org/I4210108985"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Fei Huang","raw_affiliation_strings":["Alibaba Group, Bellevue, WA, USA"],"raw_orcid":"https://orcid.org/0000-0002-3709-5053","affiliations":[{"raw_affiliation_string":"Alibaba Group, Bellevue, WA, USA","institution_ids":["https://openalex.org/I4210095624","https://openalex.org/I4210108985"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":5,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":11,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"6929","last_page":"6938"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6972851157188416},{"id":"https://openalex.org/keywords/communication-diagram","display_name":"Communication diagram","score":0.5573322176933289},{"id":"https://openalex.org/keywords/diagram","display_name":"Diagram","score":0.47877228260040283},{"id":"https://openalex.org/keywords/class-diagram","display_name":"Class diagram","score":0.4375545382499695},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.4269561469554901},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.4082998037338257},{"id":"https://openalex.org/keywords/unified-modeling-language","display_name":"Unified Modeling Language","score":0.211857408285141},{"id":"https://openalex.org/keywords/database","display_name":"Database","score":0.08943355083465576},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.0648488998413086}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6972851157188416},{"id":"https://openalex.org/C48419115","wikidata":"https://www.wikidata.org/wiki/Q950859","display_name":"Communication diagram","level":5,"score":0.5573322176933289},{"id":"https://openalex.org/C186399060","wikidata":"https://www.wikidata.org/wiki/Q959962","display_name":"Diagram","level":2,"score":0.47877228260040283},{"id":"https://openalex.org/C202446494","wikidata":"https://www.wikidata.org/wiki/Q664166","display_name":"Class diagram","level":4,"score":0.4375545382499695},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.4269561469554901},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.4082998037338257},{"id":"https://openalex.org/C145644426","wikidata":"https://www.wikidata.org/wiki/Q169411","display_name":"Unified Modeling Language","level":3,"score":0.211857408285141},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.08943355083465576},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.0648488998413086}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3664647.3681294","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3664647.3681294","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 32nd ACM International Conference on Multimedia","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":12,"referenced_works":["https://openalex.org/W1593271688","https://openalex.org/W1956340063","https://openalex.org/W2145023731","https://openalex.org/W2963420691","https://openalex.org/W3120043490","https://openalex.org/W3175569474","https://openalex.org/W4213213306","https://openalex.org/W4297207699","https://openalex.org/W4297909472","https://openalex.org/W4367367040","https://openalex.org/W4382202658","https://openalex.org/W4383604278"],"related_works":["https://openalex.org/W4242248103","https://openalex.org/W2388618932","https://openalex.org/W2296627366","https://openalex.org/W2064101236","https://openalex.org/W3144660059","https://openalex.org/W2347474403","https://openalex.org/W2374416132","https://openalex.org/W2383744988","https://openalex.org/W1536720857","https://openalex.org/W1964981556"],"abstract_inverted_index":{"Weak":[0],"diagram":[1,38,57,73,153],"analysis":[2,39,74],"abilities":[3],"of":[4,41,49,89,100],"LLMs":[5,8],"or":[6,102,131],"Multimodal":[7,42,142],"greatly":[9],"limit":[10],"their":[11],"application":[12],"scenarios":[13],"for":[14,27,76],"scientific":[15,91,152],"academic":[16,28,50],"paper":[17,29,66],"writing.":[18],"In":[19],"this":[20],"work,":[21],"towards":[22],"a":[23,55,140],"more":[24],"versatile":[25],"copilot":[26,110],"writing,":[30],"we":[31,52,70,115],"mainly":[32],"focus":[33],"on":[34,134,147],"strengthening":[35],"the":[36,65,82,98,109,112,117,120,129],"multi-modal":[37,56],"ability":[40],"LLMs.":[43],"By":[44,61],"parsing":[45],"Latex":[46,103],"source":[47],"files":[48],"papers,":[51],"carefully":[53],"build":[54],"understanding":[58,154],"dataset":[59,84,149],"M-Paper.":[60],"aligning":[62],"diagrams":[63],"in":[64,97],"with":[67,111,139],"related":[68],"paragraphs,":[69],"construct":[71],"professional":[72],"samples":[75],"training":[77,146],"and":[78,95,159],"evaluation.":[79],"M-Paper":[80],"is":[81],"first":[83],"to":[85,106],"support":[86],"joint":[87],"comprehension":[88],"multiple":[90],"diagrams,":[92],"including":[93],"figures":[94],"tables":[96],"format":[99],"images":[101],"codes.":[104],"Besides,":[105],"better":[107],"align":[108],"user's":[113],"intention,":[114],"introduce":[116],"'outline'":[118],"as":[119],"control":[121],"signal,":[122],"which":[123],"could":[124],"be":[125],"directly":[126],"given":[127],"by":[128],"user":[130],"revised":[132],"based":[133],"auto-generated":[135],"ones.":[136],"Comprehensive":[137],"experiments":[138],"state-of-the-art":[141],"LLM":[143],"demonstrate":[144],"that":[145],"our":[148],"shows":[150],"stronger":[151],"performance.":[155],"The":[156],"dataset,":[157],"code,":[158],"model":[160],"are":[161],"publicly":[162],"available":[163],"at":[164],"https://github.com/X-PLUG/mPLUG-DocOwl/tree/main/PaperOwl.":[165]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":8},{"year":2024,"cited_by_count":2}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}