{"id":"https://openalex.org/W4387969285","doi":"https://doi.org/10.1145/3581783.3612322","title":"Fine-Grained Multimodal Named Entity Recognition and Grounding with a Generative Framework","display_name":"Fine-Grained Multimodal Named Entity Recognition and Grounding with a Generative Framework","alternate_titles":[],"publication_year":2023,"publication_date":"2023-10-26","ids":{"openalex":"https://openalex.org/W4387969285","doi":"https://doi.org/10.1145/3581783.3612322"},"language":"en","primary_location":{"id":"doi:10.1145/3581783.3612322","doi":"https://doi.org/10.1145/3581783.3612322","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3581783.3612322","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3581783.3612322","source":{"id":"https://openalex.org/S4363608757","display_name":"ACM International Conference on Multimedia (ACM MM)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 31st ACM International Conference on Multimedia","raw_type":"proceedings-article"},"parent_works":[],"lineage":["https://openalex.org/W4387969285"],"child_works_count":0,"type":"conference-paper","is_traditional":true,"indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"bronze","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3581783.3612322","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5008267025","display_name":"Jieming Wang","orcid":"https://orcid.org/0000-0002-1827-1867","observed_orcids":["https://orcid.org/0000-0002-1827-1867"]},"institutions":[{"id":"https://openalex.org/I36399199","display_name":"Nanjing University of Science and Technology","ror":"https://ror.org/00xp9wg62","country_code":"CN","type":"education","lineage":["https://openalex.org/I36399199"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jieming Wang","raw_affiliation_strings":["Nanjing University of Science and Technology, Nanjing, China"],"raw_orcid":"https://orcid.org/0000-0002-1827-1867","affiliations":[{"raw_affiliation_string":"Nanjing University of Science and Technology, Nanjing, China","institution_ids":["https://openalex.org/I36399199"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5037132022","display_name":"Ziyan Li","orcid":"https://orcid.org/0009-0006-3034-7998","observed_orcids":["https://orcid.org/0009-0006-3034-7998"]},"institutions":[{"id":"https://openalex.org/I36399199","display_name":"Nanjing University of Science and Technology","ror":"https://ror.org/00xp9wg62","country_code":"CN","type":"education","lineage":["https://openalex.org/I36399199"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Ziyan Li","raw_affiliation_strings":["Nanjing University of Science and Technology, Nanjing, China"],"raw_orcid":"https://orcid.org/0009-0006-3034-7998","affiliations":[{"raw_affiliation_string":"Nanjing University of Science and Technology, Nanjing, China","institution_ids":["https://openalex.org/I36399199"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5084990301","display_name":"Jianfei Yu","orcid":"https://orcid.org/0000-0001-8380-0609","observed_orcids":["https://orcid.org/0000-0001-8380-0609"]},"institutions":[{"id":"https://openalex.org/I36399199","display_name":"Nanjing University of Science and Technology","ror":"https://ror.org/00xp9wg62","country_code":"CN","type":"education","lineage":["https://openalex.org/I36399199"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jianfei Yu","raw_affiliation_strings":["Nanjing University of Science and Technology, Nanjing, China"],"raw_orcid":"https://orcid.org/0000-0001-8380-0609","affiliations":[{"raw_affiliation_string":"Nanjing University of Science and Technology, Nanjing, China","institution_ids":["https://openalex.org/I36399199"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5022461927","display_name":"Li Yang","orcid":"https://orcid.org/0000-0001-7160-9620","observed_orcids":["https://orcid.org/0000-0001-7160-9620"]},"institutions":[{"id":"https://openalex.org/I172675005","display_name":"Nanyang Technological University","ror":"https://ror.org/02e7b5302","country_code":"SG","type":"education","lineage":["https://openalex.org/I172675005"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Li Yang","raw_affiliation_strings":["Nanyang Technological University, Singapore, Singapore"],"raw_orcid":"https://orcid.org/0000-0001-7160-9620","affiliations":[{"raw_affiliation_string":"Nanyang Technological University, Singapore, Singapore","institution_ids":["https://openalex.org/I172675005"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5035534911","display_name":"Rui Xia","orcid":"https://orcid.org/0000-0002-0621-1058","observed_orcids":["https://orcid.org/0000-0002-0621-1058"]},"institutions":[{"id":"https://openalex.org/I36399199","display_name":"Nanjing University of Science and Technology","ror":"https://ror.org/00xp9wg62","country_code":"CN","type":"education","lineage":["https://openalex.org/I36399199"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Rui Xia","raw_affiliation_strings":["Nanjing University of Science and Technology, Nanjing, China"],"raw_orcid":"https://orcid.org/0000-0002-0621-1058","affiliations":[{"raw_affiliation_string":"Nanjing University of Science and Technology, Nanjing, China","institution_ids":["https://openalex.org/I36399199"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":2.3002,"has_fulltext":true,"cited_by_count":39,"citation_normalized_percentile":{"value":0.93048066,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"3934","last_page":"3943"},"is_retracted":false,"is_pulled":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9901999831199646,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9901999831199646,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.009399999864399433,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10601","display_name":"Handwritten Text Recognition Techniques","score":9.999999747378752e-05,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/multimodal-named-entity-recognition","display_name":"multimodal named entity recognition","score":0.9919999837875366},{"id":"https://openalex.org/keywords/image-text-alignment","display_name":"image-text alignment","score":0.5820000171661377},{"id":"https://openalex.org/keywords/generative-framework","display_name":"generative framework","score":0.5360000133514404},{"id":"https://openalex.org/keywords/t5","display_name":"T5","score":0.765999972820282},{"id":"https://openalex.org/keywords/sequence-to-sequence-generation","display_name":"sequence-to-sequence generation","score":0.8370000123977661},{"id":"https://openalex.org/keywords/twitter-dataset","display_name":"Twitter dataset","score":0.7260000109672546}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8517300486564636},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.6470987796783447},{"id":"https://openalex.org/C2780801425","wikidata":"https://www.wikidata.org/wiki/Q5164392","display_name":"Construct (python library)","level":2,"score":0.6446695327758789},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6180341839790344},{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.5998082160949707},{"id":"https://openalex.org/C2778112365","wikidata":"https://www.wikidata.org/wiki/Q3511065","display_name":"Sequence (biology)","level":2,"score":0.5401870012283325},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.5245685577392578},{"id":"https://openalex.org/C2781238097","wikidata":"https://www.wikidata.org/wiki/Q175026","display_name":"Object (grammar)","level":2,"score":0.49581488966941833},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.48351719975471497},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.47344255447387695},{"id":"https://openalex.org/C2779135771","wikidata":"https://www.wikidata.org/wiki/Q403574","display_name":"Named-entity recognition","level":3,"score":0.4672144949436188},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.4574543535709381},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.44273415207862854},{"id":"https://openalex.org/C96711827","wikidata":"https://www.wikidata.org/wiki/Q17012245","display_name":"Entity linking","level":3,"score":0.41348177194595337},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.37628689408302307},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.14880189299583435},{"id":"https://openalex.org/C4554734","wikidata":"https://www.wikidata.org/wiki/Q593744","display_name":"Knowledge base","level":2,"score":0.08906260132789612},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.08291125297546387},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C120665830","wikidata":"https://www.wikidata.org/wiki/Q14620","display_name":"Optics","level":1,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C54355233","wikidata":"https://www.wikidata.org/wiki/Q7162","display_name":"Genetics","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3581783.3612322","doi":"https://doi.org/10.1145/3581783.3612322","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3581783.3612322","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3581783.3612322","source":{"id":"https://openalex.org/S4363608757","display_name":"ACM International Conference on Multimedia (ACM MM)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 31st ACM International Conference on Multimedia","raw_type":"proceedings-article"},{"id":"pmh:oai:dr.ntu.edu.sg:10356/232616","doi":null,"is_oa":false,"landing_page_url":"https://hdl.handle.net/10356/232616","pdf_url":null,"source":{"id":"https://openalex.org/S4306402609","display_name":"DR-NTU (Nanyang Technological University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I172675005","host_organization_name":"Nanyang Technological University","host_organization_lineage":["https://openalex.org/I172675005"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":null,"raw_type":"Conference Paper"}],"best_oa_location":{"id":"doi:10.1145/3581783.3612322","doi":"https://doi.org/10.1145/3581783.3612322","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3581783.3612322","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3581783.3612322","source":{"id":"https://openalex.org/S4363608757","display_name":"ACM International Conference on Multimedia (ACM MM)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 31st ACM International Conference on Multimedia","raw_type":"proceedings-article"},"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G3452498813","display_name":null,"funder_award_id":"BK20200463","funder_id":"https://openalex.org/F4320322769","funder_display_name":"Natural Science Foundation of Jiangsu Province"},{"id":"https://openalex.org/G4800195568","display_name":null,"funder_award_id":"62076133","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G6056900124","display_name":null,"funder_award_id":"62006117","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G7529710754","display_name":null,"funder_award_id":"BK20200018","funder_id":"https://openalex.org/F4320336608","funder_display_name":"Natural Science Foundation of Jiangsu Province for Distinguished Young Scholars"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320321605","display_name":"Government of Jiangsu Province","ror":"https://ror.org/004svx814"},{"id":"https://openalex.org/F4320322769","display_name":"Natural Science Foundation of Jiangsu Province","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320336608","display_name":"Natural Science Foundation of Jiangsu Province for Distinguished Young Scholars","ror":null}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4387969285.pdf","grobid_xml":"https://content.openalex.org/works/W4387969285.grobid-xml"},"referenced_works_count":60,"referenced_works":["https://openalex.org/W1975879668","https://openalex.org/W2004763266","https://openalex.org/W2096765155","https://openalex.org/W2148540243","https://openalex.org/W2153848201","https://openalex.org/W2179519966","https://openalex.org/W2250729567","https://openalex.org/W2406945108","https://openalex.org/W2588986918","https://openalex.org/W2757224794","https://openalex.org/W2757931374","https://openalex.org/W2788647998","https://openalex.org/W2789065247","https://openalex.org/W2796347433","https://openalex.org/W2798298921","https://openalex.org/W2798832674","https://openalex.org/W2799448136","https://openalex.org/W2947312908","https://openalex.org/W2947476638","https://openalex.org/W2963109634","https://openalex.org/W2963341956","https://openalex.org/W2963870853","https://openalex.org/W2964022527","https://openalex.org/W2964345792","https://openalex.org/W2982399380","https://openalex.org/W2987734933","https://openalex.org/W3011594683","https://openalex.org/W3030520226","https://openalex.org/W3035268124","https://openalex.org/W3035448883","https://openalex.org/W3035625205","https://openalex.org/W3047144459","https://openalex.org/W3092378671","https://openalex.org/W3092692431","https://openalex.org/W3093291461","https://openalex.org/W3127151332","https://openalex.org/W3156978626","https://openalex.org/W3173220247","https://openalex.org/W3173364567","https://openalex.org/W3174945605","https://openalex.org/W3175225269","https://openalex.org/W3176858586","https://openalex.org/W3177312484","https://openalex.org/W3183226462","https://openalex.org/W3199057778","https://openalex.org/W3202729335","https://openalex.org/W3207972321","https://openalex.org/W4212998232","https://openalex.org/W4221166835","https://openalex.org/W4224919571","https://openalex.org/W4225757376","https://openalex.org/W4229024390","https://openalex.org/W4233787372","https://openalex.org/W4287854428","https://openalex.org/W4304015092","https://openalex.org/W4304091743","https://openalex.org/W4312044727","https://openalex.org/W4312351586","https://openalex.org/W4382466550","https://openalex.org/W4385571080"],"related_works":["https://openalex.org/W4403487415","https://openalex.org/W4385571080","https://openalex.org/W7166878155","https://openalex.org/W4382466550","https://openalex.org/W4404782279","https://openalex.org/W7129053982","https://openalex.org/W7125985941","https://openalex.org/W7204945206","https://openalex.org/W4402906604","https://openalex.org/W4414803359"],"abstract_inverted_index":{"Multimodal":[0,78],"Named":[1,79],"Entity":[2,80],"Recognition":[3,81],"(MNER)":[4],"aims":[5,89],"to":[6,36,40,61,64,90,151],"locate":[7],"and":[8,18,82,101,118,144,184],"classify":[9,50],"named":[10,76,93],"entities":[11,28,51,94],"mentioned":[12],"in":[13,29,85,95,106],"a":[14,74,111,121,131,141,146,171],"pair":[15],"of":[16,32,173],"text":[17,33,38],"image.":[19,107],"However,":[20],"most":[21],"previous":[22],"MNER":[23,47],"works":[24],"focus":[25],"on":[26,176],"extracting":[27],"the":[30,115,137,154,177],"form":[31],"but":[34],"failing":[35],"ground":[37],"symbols":[39],"their":[41,65,97,102],"corresponding":[42],"visual":[43,104],"objects.":[44],"Moreover,":[45,108],"existing":[46],"studies":[48],"primarily":[49],"into":[52,140],"four":[53],"coarse-grained":[54],"entity":[55,99],"types,":[56,100],"which":[57,88,127],"are":[58,187],"often":[59],"insufficient":[60],"map":[62],"them":[63],"real-world":[66],"referents.":[67],"To":[68],"solve":[69],"these":[70],"limitations,":[71],"we":[72,109],"introduce":[73],"task":[75],"Fine-grained":[77],"Grounding":[83],"(FMNERG)":[84],"this":[86],"paper,":[87],"simultaneously":[91],"extract":[92],"text,":[96],"fine-grained":[98],"grounded":[103],"objects":[105],"construct":[110],"Twitter":[112,179],"dataset":[113,182],"for":[114],"FMNERG":[116,129],"task,":[117],"further":[119],"propose":[120],"T5-based":[122],"multImodal":[123],"GEneration":[124],"fRamework":[125],"(TIGER),":[126],"formulates":[128],"as":[130],"generation":[132],"problem":[133],"by":[134],"converting":[135],"all":[136],"entity-type-object":[138],"triples":[139],"target":[142,155],"sequence":[143,156],"adapts":[145],"pre-trained":[147],"sequence-to-sequence":[148],"model":[149],"T5":[150],"directly":[152],"generate":[153],"from":[157],"an":[158],"image-text":[159],"input":[160],"pair.":[161],"Experimental":[162],"results":[163],"demonstrate":[164],"that":[165],"TIGER":[166],"performs":[167],"significantly":[168],"better":[169],"than":[170],"number":[172],"baseline":[174],"systems":[175],"annotated":[178],"dataset.":[180],"Our":[181],"annotation":[183],"source":[185],"code":[186],"publicly":[188],"released":[189],"at":[190],"https://github.com/NUSTM/FMNERG.":[191]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W4387969285","counts_by_year":[{"year":2026,"cited_by_count":18},{"year":2025,"cited_by_count":14},{"year":2024,"cited_by_count":7}],"updated_date":"2026-11-17T12:49:18.490984","created_date":"2023-10-28T00:00:00"}
