{"id":"https://openalex.org/W7129053982","doi":"https://doi.org/10.1109/tmm.2026.3664998","title":"Advancing Grounded Multimodal Named Entity Recognition via LLM-Based Reformulation and Box-Based Segmentation","display_name":"Advancing Grounded Multimodal Named Entity Recognition via LLM-Based Reformulation and Box-Based Segmentation","publication_year":2026,"publication_date":"2026-01-01","ids":{"openalex":"https://openalex.org/W7129053982","doi":"https://doi.org/10.1109/tmm.2026.3664998"},"language":null,"primary_location":{"id":"doi:10.1109/tmm.2026.3664998","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tmm.2026.3664998","pdf_url":null,"source":{"id":"https://openalex.org/S137030581","display_name":"IEEE Transactions on Multimedia","issn_l":"1520-9210","issn":["1520-9210","1941-0077"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Multimedia","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100601056","display_name":"Jinyuan Li","orcid":"https://orcid.org/0009-0001-3343-2497"},"institutions":[{"id":"https://openalex.org/I162868743","display_name":"Tianjin University","ror":"https://ror.org/012tb2g32","country_code":"CN","type":"education","lineage":["https://openalex.org/I162868743"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jinyuan Li","raw_affiliation_strings":["College of Intelligence and Computing, Tianjin University, Tianjin, China"],"raw_orcid":"https://orcid.org/0009-0001-3343-2497","affiliations":[{"raw_affiliation_string":"College of Intelligence and Computing, Tianjin University, Tianjin, China","institution_ids":["https://openalex.org/I162868743"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5126165367","display_name":"Ziyan Li","orcid":null},"institutions":[{"id":"https://openalex.org/I36399199","display_name":"Nanjing University of Science and Technology","ror":"https://ror.org/00xp9wg62","country_code":"CN","type":"education","lineage":["https://openalex.org/I36399199"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Ziyan Li","raw_affiliation_strings":["Nanjing University of Science and Technology, Nanjing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Nanjing University of Science and Technology, Nanjing, China","institution_ids":["https://openalex.org/I36399199"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101277553","display_name":"Han Li","orcid":"https://orcid.org/0000-0001-9368-826X"},"institutions":[{"id":"https://openalex.org/I9086337","display_name":"Taiyuan University of Technology","ror":"https://ror.org/03kv08d37","country_code":"CN","type":"education","lineage":["https://openalex.org/I9086337"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Han Li","raw_affiliation_strings":["College of Mathematics, Taiyuan University of Technology, Taiyuan, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"College of Mathematics, Taiyuan University of Technology, Taiyuan, China","institution_ids":["https://openalex.org/I9086337"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5084990301","display_name":"Jianfei Yu","orcid":"https://orcid.org/0000-0001-8380-0609"},"institutions":[{"id":"https://openalex.org/I36399199","display_name":"Nanjing University of Science and Technology","ror":"https://ror.org/00xp9wg62","country_code":"CN","type":"education","lineage":["https://openalex.org/I36399199"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jianfei Yu","raw_affiliation_strings":["Nanjing University of Science and Technology, Nanjing, China"],"raw_orcid":"https://orcid.org/0000-0001-8380-0609","affiliations":[{"raw_affiliation_string":"Nanjing University of Science and Technology, Nanjing, China","institution_ids":["https://openalex.org/I36399199"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5126091416","display_name":"Rui Xia","orcid":null},"institutions":[{"id":"https://openalex.org/I36399199","display_name":"Nanjing University of Science and Technology","ror":"https://ror.org/00xp9wg62","country_code":"CN","type":"education","lineage":["https://openalex.org/I36399199"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Rui Xia","raw_affiliation_strings":["Nanjing University of Science and Technology, Nanjing, China"],"raw_orcid":"https://orcid.org/0000-0002-0621-1058","affiliations":[{"raw_affiliation_string":"Nanjing University of Science and Technology, Nanjing, China","institution_ids":["https://openalex.org/I36399199"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Di Sun","orcid":"https://orcid.org/0000-0003-2793-7066"},"institutions":[{"id":"https://openalex.org/I132369690","display_name":"Tianjin University of Science and Technology","ror":"https://ror.org/018rbtf37","country_code":"CN","type":"education","lineage":["https://openalex.org/I132369690"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Di Sun","raw_affiliation_strings":["Tianjin University of Science and Technology, Tianjin, China"],"raw_orcid":"https://orcid.org/0000-0003-2793-7066","affiliations":[{"raw_affiliation_string":"Tianjin University of Science and Technology, Tianjin, China","institution_ids":["https://openalex.org/I132369690"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5126157053","display_name":"Gang Pan","orcid":null},"institutions":[{"id":"https://openalex.org/I162868743","display_name":"Tianjin University","ror":"https://ror.org/012tb2g32","country_code":"CN","type":"education","lineage":["https://openalex.org/I162868743"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Gang Pan","raw_affiliation_strings":["College of Intelligence and Computing, Tianjin University, Tianjin, China"],"raw_orcid":"https://orcid.org/0000-0003-2155-4689","affiliations":[{"raw_affiliation_string":"College of Intelligence and Computing, Tianjin University, Tianjin, China","institution_ids":["https://openalex.org/I162868743"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":9.2305,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":{"value":0.95997474,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":97},"biblio":{"volume":"28","issue":null,"first_page":"5422","last_page":"5436"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.8687999844551086,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.8687999844551086,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.06790000200271606,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11273","display_name":"Advanced Graph Neural Networks","score":0.012299999594688416,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.6263999938964844},{"id":"https://openalex.org/keywords/named-entity-recognition","display_name":"Named-entity recognition","score":0.6241000294685364},{"id":"https://openalex.org/keywords/construct","display_name":"Construct (python library)","score":0.5767999887466431},{"id":"https://openalex.org/keywords/minimum-bounding-box","display_name":"Minimum bounding box","score":0.5752999782562256},{"id":"https://openalex.org/keywords/textual-entailment","display_name":"Textual entailment","score":0.5522000193595886},{"id":"https://openalex.org/keywords/segmentation","display_name":"Segmentation","score":0.5371000170707703},{"id":"https://openalex.org/keywords/ambiguity","display_name":"Ambiguity","score":0.5110999941825867},{"id":"https://openalex.org/keywords/entity-linking","display_name":"Entity linking","score":0.499099999666214},{"id":"https://openalex.org/keywords/phrase","display_name":"Phrase","score":0.4885999858379364},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.47429999709129333}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.878000020980835},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6547999978065491},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.6263999938964844},{"id":"https://openalex.org/C2779135771","wikidata":"https://www.wikidata.org/wiki/Q403574","display_name":"Named-entity recognition","level":3,"score":0.6241000294685364},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.5827999711036682},{"id":"https://openalex.org/C2780801425","wikidata":"https://www.wikidata.org/wiki/Q5164392","display_name":"Construct (python library)","level":2,"score":0.5767999887466431},{"id":"https://openalex.org/C147037132","wikidata":"https://www.wikidata.org/wiki/Q6865426","display_name":"Minimum bounding box","level":3,"score":0.5752999782562256},{"id":"https://openalex.org/C95318506","wikidata":"https://www.wikidata.org/wiki/Q6588467","display_name":"Textual entailment","level":3,"score":0.5522000193595886},{"id":"https://openalex.org/C89600930","wikidata":"https://www.wikidata.org/wiki/Q1423946","display_name":"Segmentation","level":2,"score":0.5371000170707703},{"id":"https://openalex.org/C2780522230","wikidata":"https://www.wikidata.org/wiki/Q1140419","display_name":"Ambiguity","level":2,"score":0.5110999941825867},{"id":"https://openalex.org/C96711827","wikidata":"https://www.wikidata.org/wiki/Q17012245","display_name":"Entity linking","level":3,"score":0.499099999666214},{"id":"https://openalex.org/C2776224158","wikidata":"https://www.wikidata.org/wiki/Q187931","display_name":"Phrase","level":2,"score":0.4885999858379364},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.47429999709129333},{"id":"https://openalex.org/C2781238097","wikidata":"https://www.wikidata.org/wiki/Q175026","display_name":"Object (grammar)","level":2,"score":0.46939998865127563},{"id":"https://openalex.org/C22367795","wikidata":"https://www.wikidata.org/wiki/Q7625208","display_name":"Structured prediction","level":2,"score":0.44269999861717224},{"id":"https://openalex.org/C63584917","wikidata":"https://www.wikidata.org/wiki/Q333286","display_name":"Bounding overwatch","level":2,"score":0.42480000853538513},{"id":"https://openalex.org/C153962237","wikidata":"https://www.wikidata.org/wiki/Q1401131","display_name":"Noun phrase","level":3,"score":0.38510000705718994},{"id":"https://openalex.org/C152565575","wikidata":"https://www.wikidata.org/wiki/Q1124538","display_name":"Conditional random field","level":2,"score":0.3544999957084656},{"id":"https://openalex.org/C81917197","wikidata":"https://www.wikidata.org/wiki/Q628760","display_name":"Selection (genetic algorithm)","level":2,"score":0.33980000019073486},{"id":"https://openalex.org/C44291984","wikidata":"https://www.wikidata.org/wiki/Q1074173","display_name":"Question answering","level":2,"score":0.31220000982284546},{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.3052999973297119},{"id":"https://openalex.org/C175154964","wikidata":"https://www.wikidata.org/wiki/Q380077","display_name":"Task analysis","level":3,"score":0.30239999294281006},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.30070000886917114},{"id":"https://openalex.org/C134752490","wikidata":"https://www.wikidata.org/wiki/Q374182","display_name":"Logical consequence","level":2,"score":0.29789999127388},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.29420000314712524},{"id":"https://openalex.org/C64876066","wikidata":"https://www.wikidata.org/wiki/Q5141226","display_name":"Cognitive neuroscience of visual object recognition","level":3,"score":0.29179999232292175},{"id":"https://openalex.org/C157486923","wikidata":"https://www.wikidata.org/wiki/Q1376436","display_name":"String (physics)","level":2,"score":0.2872999906539917},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.2745000123977661},{"id":"https://openalex.org/C2777508537","wikidata":"https://www.wikidata.org/wiki/Q7936620","display_name":"Visual reasoning","level":2,"score":0.2662000060081482},{"id":"https://openalex.org/C2776151529","wikidata":"https://www.wikidata.org/wiki/Q3045304","display_name":"Object detection","level":3,"score":0.26089999079704285},{"id":"https://openalex.org/C67186912","wikidata":"https://www.wikidata.org/wiki/Q367664","display_name":"Data modeling","level":2,"score":0.2574999928474426},{"id":"https://openalex.org/C184898388","wikidata":"https://www.wikidata.org/wiki/Q1435712","display_name":"Pairwise comparison","level":2,"score":0.2547999918460846},{"id":"https://openalex.org/C124504099","wikidata":"https://www.wikidata.org/wiki/Q56933","display_name":"Image segmentation","level":3,"score":0.25450000166893005},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.250900000333786}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tmm.2026.3664998","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tmm.2026.3664998","pdf_url":null,"source":{"id":"https://openalex.org/S137030581","display_name":"IEEE Transactions on Multimedia","issn_l":"1520-9210","issn":["1520-9210","1941-0077"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Multimedia","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Quality Education","score":0.4015849232673645,"id":"https://metadata.un.org/sdg/4"}],"awards":[{"id":"https://openalex.org/G8902784279","display_name":null,"funder_award_id":"62576243","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":56,"referenced_works":["https://openalex.org/W1773149199","https://openalex.org/W1861492603","https://openalex.org/W1975879668","https://openalex.org/W2108598243","https://openalex.org/W2194775991","https://openalex.org/W2247513039","https://openalex.org/W2251512949","https://openalex.org/W2277195237","https://openalex.org/W2489434015","https://openalex.org/W2745461083","https://openalex.org/W2757389665","https://openalex.org/W2788647998","https://openalex.org/W2798298921","https://openalex.org/W2886641317","https://openalex.org/W2888975113","https://openalex.org/W2896457183","https://openalex.org/W2948427397","https://openalex.org/W2948672349","https://openalex.org/W2962902328","https://openalex.org/W2962982907","https://openalex.org/W2963150697","https://openalex.org/W2963625095","https://openalex.org/W2964284374","https://openalex.org/W3035390927","https://openalex.org/W3035448883","https://openalex.org/W3047144459","https://openalex.org/W3173220247","https://openalex.org/W3175225269","https://openalex.org/W3176641147","https://openalex.org/W3176858586","https://openalex.org/W3180169285","https://openalex.org/W3199693760","https://openalex.org/W3209114016","https://openalex.org/W4225757376","https://openalex.org/W4293518017","https://openalex.org/W4304015092","https://openalex.org/W4304091743","https://openalex.org/W4309181071","https://openalex.org/W4312578467","https://openalex.org/W4382999323","https://openalex.org/W4385571080","https://openalex.org/W4385573951","https://openalex.org/W4386065596","https://openalex.org/W4386065815","https://openalex.org/W4386075493","https://openalex.org/W4387969106","https://openalex.org/W4387969285","https://openalex.org/W4389523688","https://openalex.org/W4389523957","https://openalex.org/W4390873343","https://openalex.org/W4390874575","https://openalex.org/W4402684284","https://openalex.org/W4403577400","https://openalex.org/W4411120260","https://openalex.org/W4412888145","https://openalex.org/W7131792124"],"related_works":[],"abstract_inverted_index":{"Grounded":[0],"Multimodal":[1,189],"Named":[2,190],"Entity":[3,140,154,191],"Recognition":[4,192],"(GMNER)":[5],"task":[6,20,194],"aims":[7],"to":[8,37,104,117,169,220,228],"identify":[9],"named":[10,42,64],"entities,":[11],"entity":[12],"types":[13],"and":[14,31,62,84,113,144,153,165,195,205,210,249,253],"their":[15],"corresponding":[16,196],"visual":[17],"regions.":[18],"GMNER":[19,134,223],"exhibits":[21],"two":[22,98,129],"challenging":[23],"attributes:":[24],"1)":[25,100],"The":[26,137],"tenuous":[27],"correlation":[28],"between":[29,51],"images":[30],"text":[32],"on":[33,242],"social":[34],"media":[35],"contributes":[36],"a":[38,49,72,78],"notable":[39],"proportion":[40],"of":[41,132,139,212],"entities":[43],"being":[44],"ungroundable.":[45],"2)":[46,136],"There":[47],"exists":[48],"distinction":[50],"coarse-grained":[52,177],"noun":[53],"phrases":[54],"used":[55],"in":[56,181],"similar":[57],"tasks":[58],"(e.g.,":[59],"phrase":[60],"localization)":[61],"fine-grained":[63,202],"entities.":[65],"In":[66],"this":[67],"paper,":[68],"we":[69,183],"propose":[70],"RiVEG,":[71],"unified":[73],"framework":[74,161],"that":[75,236],"reformulates":[76],"GMNERinto":[77],"joint":[79],"modeling":[80],"paradigm":[81],"spanning":[82],"MNER,VE,":[83],"VGperspectives":[85],"by":[86],"leveraging":[87],"large":[88],"language":[89],"models":[90],"(LLMs)":[91],"as":[92],"connecting":[93],"bridges.":[94],"This":[95,157],"reformulation":[96],"brings":[97],"benefits:":[99],"It":[101],"enables":[102],"us":[103],"optimize":[105],"the":[106,115,128,159,171,176,186,208,226,230,246],"MNER":[107,111],"module":[108,143,148],"for":[109],"optimal":[110],"performance":[112],"eliminates":[114],"need":[116],"pre-extract":[118],"region":[119],"features":[120],"using":[121,213],"object":[122],"detection":[123],"methods,":[124],"thus":[125],"naturally":[126],"addressing":[127],"major":[130],"limitations":[131],"existing":[133],"methods.":[135],"introduction":[138],"Expansion":[141],"Expression":[142],"Visual":[145,150],"Entailment":[146],"(VE)":[147],"unifies":[149],"Grounding":[151,155],"(VG)":[152],"(EG).":[156],"endows":[158],"proposed":[160],"with":[162,225],"unlimited":[163],"data":[164],"model":[166,224],"scalability.":[167],"Furthermore,":[168],"address":[170],"potential":[172],"ambiguity":[173],"stemming":[174],"from":[175],"bounding":[178],"box":[179,214],"output":[180],"GMNER,":[182,248],"further":[184],"construct":[185],"new":[187],"Segmented":[188],"(SMNER)":[193],"Twitter-SMNER":[197],"dataset":[198],"aimed":[199],"at":[200],"generating":[201],"segmentation":[203],"masks,":[204],"experimentally":[206],"demonstrate":[207,235],"feasibility":[209],"effectiveness":[211],"prompt-based":[215],"Segment":[216],"Anything":[217],"Model":[218],"(SAM)":[219],"empower":[221],"any":[222],"ability":[227],"accomplish":[229],"SMNER":[231,250],"task.":[232],"Extensive":[233],"experiments":[234],"RiVEG":[237],"significantly":[238],"outperforms":[239],"SoTA":[240],"methods":[241],"four":[243],"datasets":[244],"across":[245],"MNER,":[247],"tasks.":[251],"Datasets":[252],"Code":[254],"will":[255],"be":[256],"released":[257],"athttps://github.com/JinYuanLi0012/RiVEG.":[258]},"counts_by_year":[{"year":2026,"cited_by_count":1}],"updated_date":"2026-07-29T09:40:50.615796","created_date":"2026-02-17T00:00:00"}