{"id":"https://openalex.org/W4402353753","doi":"https://doi.org/10.1109/ijcnn60899.2024.10650770","title":"Exploiting Visual Relation and Multi-Grained Knowledge for Multimodal Relation Extraction","display_name":"Exploiting Visual Relation and Multi-Grained Knowledge for Multimodal Relation Extraction","publication_year":2024,"publication_date":"2024-06-30","ids":{"openalex":"https://openalex.org/W4402353753","doi":"https://doi.org/10.1109/ijcnn60899.2024.10650770"},"language":"en","primary_location":{"id":"doi:10.1109/ijcnn60899.2024.10650770","is_oa":false,"landing_page_url":"http://dx.doi.org/10.1109/ijcnn60899.2024.10650770","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2024 International Joint Conference on Neural Networks (IJCNN)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100599105","display_name":"Qianru Shen","orcid":null},"institutions":[{"id":"https://openalex.org/I19820366","display_name":"Chinese Academy of Sciences","ror":"https://ror.org/034t30j35","country_code":"CN","type":"government","lineage":["https://openalex.org/I19820366"]},{"id":"https://openalex.org/I4210156404","display_name":"Institute of Information Engineering","ror":"https://ror.org/04r53se39","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210156404"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Qianru Shen","raw_affiliation_strings":["Chinese Academy of Sciences,Institute of Information Engineering,Beijing,China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Chinese Academy of Sciences,Institute of Information Engineering,Beijing,China","institution_ids":["https://openalex.org/I19820366","https://openalex.org/I4210156404"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5113893497","display_name":"Hailun Lin","orcid":null},"institutions":[{"id":"https://openalex.org/I19820366","display_name":"Chinese Academy of Sciences","ror":"https://ror.org/034t30j35","country_code":"CN","type":"government","lineage":["https://openalex.org/I19820366"]},{"id":"https://openalex.org/I4210156404","display_name":"Institute of Information Engineering","ror":"https://ror.org/04r53se39","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210156404"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Hailun Lin","raw_affiliation_strings":["Chinese Academy of Sciences,Institute of Information Engineering,Beijing,China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Chinese Academy of Sciences,Institute of Information Engineering,Beijing,China","institution_ids":["https://openalex.org/I19820366","https://openalex.org/I4210156404"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5108048937","display_name":"Huan Liu","orcid":"https://orcid.org/0000-0002-0368-2196"},"institutions":[{"id":"https://openalex.org/I19820366","display_name":"Chinese Academy of Sciences","ror":"https://ror.org/034t30j35","country_code":"CN","type":"government","lineage":["https://openalex.org/I19820366"]},{"id":"https://openalex.org/I4210156404","display_name":"Institute of Information Engineering","ror":"https://ror.org/04r53se39","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210156404"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Huan Liu","raw_affiliation_strings":["Chinese Academy of Sciences,Institute of Information Engineering,Beijing,China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Chinese Academy of Sciences,Institute of Information Engineering,Beijing,China","institution_ids":["https://openalex.org/I19820366","https://openalex.org/I4210156404"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5078642643","display_name":"Lin Zheng","orcid":"https://orcid.org/0000-0002-8057-4949"},"institutions":[{"id":"https://openalex.org/I19820366","display_name":"Chinese Academy of Sciences","ror":"https://ror.org/034t30j35","country_code":"CN","type":"government","lineage":["https://openalex.org/I19820366"]},{"id":"https://openalex.org/I4210156404","display_name":"Institute of Information Engineering","ror":"https://ror.org/04r53se39","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210156404"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zheng Lin","raw_affiliation_strings":["Chinese Academy of Sciences,Institute of Information Engineering,Beijing,China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Chinese Academy of Sciences,Institute of Information Engineering,Beijing,China","institution_ids":["https://openalex.org/I19820366","https://openalex.org/I4210156404"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5115591448","display_name":"Weiping Wang","orcid":null},"institutions":[{"id":"https://openalex.org/I19820366","display_name":"Chinese Academy of Sciences","ror":"https://ror.org/034t30j35","country_code":"CN","type":"government","lineage":["https://openalex.org/I19820366"]},{"id":"https://openalex.org/I4210156404","display_name":"Institute of Information Engineering","ror":"https://ror.org/04r53se39","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210156404"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Weiping Wang","raw_affiliation_strings":["Chinese Academy of Sciences,Institute of Information Engineering,Beijing,China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Chinese Academy of Sciences,Institute of Information Engineering,Beijing,China","institution_ids":["https://openalex.org/I19820366","https://openalex.org/I4210156404"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":"35","issue":null,"first_page":"1","last_page":"8"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9983000159263611,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9975000023841858,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/relation","display_name":"Relation (database)","score":0.8444128036499023},{"id":"https://openalex.org/keywords/relationship-extraction","display_name":"Relationship extraction","score":0.7875617742538452},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7773107290267944},{"id":"https://openalex.org/keywords/extraction","display_name":"Extraction (chemistry)","score":0.4858803153038025},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.4112253785133362},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3640119731426239},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.19160184264183044}],"concepts":[{"id":"https://openalex.org/C25343380","wikidata":"https://www.wikidata.org/wiki/Q277521","display_name":"Relation (database)","level":2,"score":0.8444128036499023},{"id":"https://openalex.org/C153604712","wikidata":"https://www.wikidata.org/wiki/Q7310755","display_name":"Relationship extraction","level":3,"score":0.7875617742538452},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7773107290267944},{"id":"https://openalex.org/C4725764","wikidata":"https://www.wikidata.org/wiki/Q844704","display_name":"Extraction (chemistry)","level":2,"score":0.4858803153038025},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.4112253785133362},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3640119731426239},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.19160184264183044},{"id":"https://openalex.org/C43617362","wikidata":"https://www.wikidata.org/wiki/Q170050","display_name":"Chromatography","level":1,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/ijcnn60899.2024.10650770","is_oa":false,"landing_page_url":"http://dx.doi.org/10.1109/ijcnn60899.2024.10650770","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2024 International Joint Conference on Neural Networks (IJCNN)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320321133","display_name":"Chinese Academy of Sciences","ror":"https://ror.org/034t30j35"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":44,"referenced_works":["https://openalex.org/W2077069816","https://openalex.org/W2251135946","https://openalex.org/W2251622960","https://openalex.org/W2277195237","https://openalex.org/W2506483933","https://openalex.org/W2560674852","https://openalex.org/W2896457183","https://openalex.org/W2953072307","https://openalex.org/W2962716332","https://openalex.org/W2963101956","https://openalex.org/W2963536419","https://openalex.org/W2964022985","https://openalex.org/W2964187781","https://openalex.org/W2966715458","https://openalex.org/W2968124245","https://openalex.org/W3034835156","https://openalex.org/W3166170409","https://openalex.org/W3177287302","https://openalex.org/W3207972321","https://openalex.org/W4205509257","https://openalex.org/W4226278401","https://openalex.org/W4229024390","https://openalex.org/W4231449374","https://openalex.org/W4285276085","https://openalex.org/W4285294723","https://openalex.org/W4287854428","https://openalex.org/W4322718191","https://openalex.org/W4323717348","https://openalex.org/W4365802806","https://openalex.org/W4366352717","https://openalex.org/W4382317990","https://openalex.org/W4385572347","https://openalex.org/W4385573951","https://openalex.org/W4386555477","https://openalex.org/W4388585881","https://openalex.org/W6766904570","https://openalex.org/W6767279747","https://openalex.org/W6791353385","https://openalex.org/W6810738896","https://openalex.org/W6849177959","https://openalex.org/W6850071225","https://openalex.org/W6850625674","https://openalex.org/W6855707979","https://openalex.org/W6858522248"],"related_works":["https://openalex.org/W2976808399","https://openalex.org/W2609844752","https://openalex.org/W2981341912","https://openalex.org/W4285246823","https://openalex.org/W4226278302","https://openalex.org/W4385734297","https://openalex.org/W4221160509","https://openalex.org/W2547211086","https://openalex.org/W2538200646","https://openalex.org/W2888033806"],"abstract_inverted_index":{"Given":[0],"a":[1,80,99,124,179],"text":[2,194],"and":[3,74,109,145,164,182,192,231],"its":[4],"related":[5,88],"image,":[6],"the":[7,16,24,60,75,113,117,129,150,153,168,187,197,202],"multimodal":[8,126],"relation":[9,19,199],"extraction":[10],"(MRE)":[11],"task":[12],"aims":[13],"at":[14],"predicting":[15],"correct":[17,61],"semantic":[18],"between":[20,49],"two":[21,41],"entities":[22,73],"in":[23,216],"input":[25,76,118,151],"text.":[26],"Though":[27],"certain":[28],"advances":[29],"have":[30],"been":[31],"made":[32],"by":[33,178,213],"recent":[34,207],"MRE":[35,114],"approaches,":[36],"they":[37,44,64],"still":[38],"suffer":[39],"from":[40],"drawbacks.":[42],"First,":[43],"ignore":[45],"fine-grained":[46],"visual":[47,144],"relations":[48,142],"image":[50],"objects":[51],"which":[52,104],"can":[53],"serve":[54],"as":[55],"important":[56],"clues":[57],"for":[58,112,235],"inferring":[59],"relation.":[62],"Second,":[63],"are":[65],"unable":[66],"to":[67,79,134,161,195,225],"utilize":[68],"useful":[69,233],"external":[70],"knowledge":[71,166,174],"about":[72],"sentence,":[77],"leading":[78],"sub-optimal":[81],"performance":[82],"when":[83],"processing":[84],"examples":[85],"that":[86],"demand":[87],"background":[89],"or":[90],"commonsense":[91],"knowledge.":[92],"To":[93],"alleviate":[94],"above":[95],"limitations,":[96],"we":[97],"propose":[98],"novel":[100],"method,":[101],"named":[102],"VRMK,":[103],"exploits":[105],"both":[106],"Visual":[107],"Relations":[108],"Multi-grained":[110],"Knowledge":[111],"task.":[115],"Specifically,":[116],"image-text":[119],"pair":[120],"is":[121,132,159,176,183],"converted":[122],"into":[123],"unified":[125],"graph.":[127],"Then,":[128],"relation-aware":[130],"transformer":[131],"adopted":[133],"update":[135],"node":[136],"representations":[137,188],"while":[138],"explicitly":[139],"encoding":[140],"diverse":[141],"among":[143],"textual":[146],"nodes.":[147],"Based":[148],"on":[149],"text,":[152],"powerful":[154],"large":[155],"language":[156],"model":[157],"(LLM)":[158],"used":[160],"generate":[162],"entity-level":[163],"sentence-level":[165],"with":[167,186],"in-context":[169],"learning.":[170],"The":[171],"most":[172],"relevant":[173],"information":[175],"captured":[177],"cross-attention":[180],"mechanism":[181],"further":[184],"combined":[185],"of":[189,228],"entity":[190],"nodes":[191],"original":[193],"predict":[196],"final":[198],"label.":[200],"On":[201],"MNRE":[203],"dataset,":[204],"VRMK":[205],"outperforms":[206],"state-of-the-art":[208],"baselines":[209],"including":[210],"LLM-based":[211],"methods":[212],"2.71%":[214],"(82.55%\u219285.26%":[215],"F1":[217],"score).":[218],"We":[219],"also":[220],"conduct":[221],"extensive":[222],"ablation":[223],"experiments":[224],"reveal":[226],"contributions":[227],"different":[229],"modules":[230],"provide":[232],"insights":[234],"future":[236],"research.":[237]},"counts_by_year":[{"year":2026,"cited_by_count":1}],"updated_date":"2026-07-22T07:51:19.307946","created_date":"2025-10-10T00:00:00"}