{"id":"https://openalex.org/W4403577938","doi":"https://doi.org/10.1145/3627673.3679680","title":"MSKR: Advancing Multi-modal Structured Knowledge Representation with Synergistic Hard Negative Samples","display_name":"MSKR: Advancing Multi-modal Structured Knowledge Representation with Synergistic Hard Negative Samples","publication_year":2024,"publication_date":"2024-10-20","ids":{"openalex":"https://openalex.org/W4403577938","doi":"https://doi.org/10.1145/3627673.3679680"},"language":"en","primary_location":{"id":"doi:10.1145/3627673.3679680","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3627673.3679680","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 33rd ACM International Conference on Information and Knowledge Management","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3627673.3679680","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5056523030","display_name":"S. F. Zhang","orcid":null},"institutions":[{"id":"https://openalex.org/I4210156404","display_name":"Institute of Information Engineering","ror":"https://ror.org/04r53se39","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210156404"]},{"id":"https://openalex.org/I4210165038","display_name":"University of Chinese Academy of Sciences","ror":"https://ror.org/05qbk4x57","country_code":"CN","type":"education","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210165038"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Shuili Zhang","raw_affiliation_strings":["Institute of Information Engineering, Chinese Academy of Sciences & School of Cyber Security, University of Chinese Academy of Sciences, Beijing, China"],"affiliations":[{"raw_affiliation_string":"Institute of Information Engineering, Chinese Academy of Sciences & School of Cyber Security, University of Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I4210156404","https://openalex.org/I4210165038"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5042345951","display_name":"Hongzhang Mu","orcid":"https://orcid.org/0000-0003-0378-4587"},"institutions":[{"id":"https://openalex.org/I4210156404","display_name":"Institute of Information Engineering","ror":"https://ror.org/04r53se39","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210156404"]},{"id":"https://openalex.org/I4210165038","display_name":"University of Chinese Academy of Sciences","ror":"https://ror.org/05qbk4x57","country_code":"CN","type":"education","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210165038"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Hongzhang Mu","raw_affiliation_strings":["Institute of Information Engineering, Chinese Academy of Sciences & School of Cyber Security, University of Chinese Academy of Sciences, Beijing, China"],"affiliations":[{"raw_affiliation_string":"Institute of Information Engineering, Chinese Academy of Sciences & School of Cyber Security, University of Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I4210156404","https://openalex.org/I4210165038"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5080686378","display_name":"Tingwen Liu","orcid":"https://orcid.org/0000-0002-0750-6923"},"institutions":[{"id":"https://openalex.org/I4210156404","display_name":"Institute of Information Engineering","ror":"https://ror.org/04r53se39","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210156404"]},{"id":"https://openalex.org/I4210165038","display_name":"University of Chinese Academy of Sciences","ror":"https://ror.org/05qbk4x57","country_code":"CN","type":"education","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210165038"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Tingwen Liu","raw_affiliation_strings":["Institute of Information Engineering, Chinese Academy of Sciences & School of Cyber Security, University of Chinese Academy of Sciences, Beijing, China"],"affiliations":[{"raw_affiliation_string":"Institute of Information Engineering, Chinese Academy of Sciences & School of Cyber Security, University of Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I4210156404","https://openalex.org/I4210165038"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5104636147","display_name":"Qianqian Tong","orcid":"https://orcid.org/0000-0003-1617-012X"},"institutions":[{"id":"https://openalex.org/I4210136793","display_name":"Peng Cheng Laboratory","ror":"https://ror.org/03qdqbt06","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4210136793"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Qianqian Tong","raw_affiliation_strings":["Department of Strategic and Advanced Interdisciplinary Research, Peng Cheng Laboratory, Shenzhen, China"],"affiliations":[{"raw_affiliation_string":"Department of Strategic and Advanced Interdisciplinary Research, Peng Cheng Laboratory, Shenzhen, China","institution_ids":["https://openalex.org/I4210136793"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5080665569","display_name":"Jiawei Sheng","orcid":"https://orcid.org/0000-0002-4865-982X"},"institutions":[{"id":"https://openalex.org/I19820366","display_name":"Chinese Academy of Sciences","ror":"https://ror.org/034t30j35","country_code":"CN","type":"government","lineage":["https://openalex.org/I19820366"]},{"id":"https://openalex.org/I4210156404","display_name":"Institute of Information Engineering","ror":"https://ror.org/04r53se39","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210156404"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jiawei Sheng","raw_affiliation_strings":["Institute of Information Engineering, Chinese Academy of Sciences, Beijing, China"],"affiliations":[{"raw_affiliation_string":"Institute of Information Engineering, Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I4210156404","https://openalex.org/I19820366"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5056523030"],"corresponding_institution_ids":["https://openalex.org/I4210156404","https://openalex.org/I4210165038"],"apc_list":null,"apc_paid":null,"fwci":0.6813,"has_fulltext":false,"cited_by_count":2,"citation_normalized_percentile":{"value":0.76090236,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":91,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"3207","last_page":"3216"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/modal","display_name":"Modal","score":0.6736757159233093},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6512480974197388},{"id":"https://openalex.org/keywords/representation","display_name":"Representation (politics)","score":0.596246063709259},{"id":"https://openalex.org/keywords/knowledge-management","display_name":"Knowledge management","score":0.393303781747818},{"id":"https://openalex.org/keywords/materials-science","display_name":"Materials science","score":0.1379581093788147}],"concepts":[{"id":"https://openalex.org/C71139939","wikidata":"https://www.wikidata.org/wiki/Q910194","display_name":"Modal","level":2,"score":0.6736757159233093},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6512480974197388},{"id":"https://openalex.org/C2776359362","wikidata":"https://www.wikidata.org/wiki/Q2145286","display_name":"Representation (politics)","level":3,"score":0.596246063709259},{"id":"https://openalex.org/C56739046","wikidata":"https://www.wikidata.org/wiki/Q192060","display_name":"Knowledge management","level":1,"score":0.393303781747818},{"id":"https://openalex.org/C192562407","wikidata":"https://www.wikidata.org/wiki/Q228736","display_name":"Materials science","level":0,"score":0.1379581093788147},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C188027245","wikidata":"https://www.wikidata.org/wiki/Q750446","display_name":"Polymer chemistry","level":1,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C94625758","wikidata":"https://www.wikidata.org/wiki/Q7163","display_name":"Politics","level":2,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3627673.3679680","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3627673.3679680","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 33rd ACM International Conference on Information and Knowledge Management","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3627673.3679680","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3627673.3679680","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 33rd ACM International Conference on Information and Knowledge Management","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":16,"referenced_works":["https://openalex.org/W1554540371","https://openalex.org/W1861492603","https://openalex.org/W2081580037","https://openalex.org/W2740887992","https://openalex.org/W2964187781","https://openalex.org/W2981852735","https://openalex.org/W3034999214","https://openalex.org/W3173909648","https://openalex.org/W4288089799","https://openalex.org/W4312261477","https://openalex.org/W4312420092","https://openalex.org/W4312910992","https://openalex.org/W4312933868","https://openalex.org/W4384661029","https://openalex.org/W4386072101","https://openalex.org/W4393154896"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W4391913857","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W4396696052"],"abstract_inverted_index":{"Despite":[0],"the":[1,65,90,108,121,144,161,172,187],"notable":[2],"progress":[3],"achieved":[4],"by":[5],"large-scale":[6],"vision-language":[7],"pre-training":[8],"models":[9],"in":[10,22,56,63,112,181,200,209],"a":[11,75,101],"wide":[12],"range":[13],"of":[14,31],"multi-modal":[15],"tasks,":[16],"their":[17,118,139],"performance":[18],"often":[19],"falls":[20],"short":[21],"image-text":[23],"matching":[24,92],"challenges":[25],"that":[26,42,168,194],"require":[27],"an":[28],"in-depth":[29],"understanding":[30],"structured":[32,49,102,109,129,182,201],"representations.":[33],"For":[34],"instance,":[35],"when":[36],"distinguishing":[37],"between":[38],"texts":[39],"or":[40,58],"images":[41],"are":[43,68],"generally":[44],"similar":[45],"but":[46,204],"have":[47],"distinct":[48],"knowledge":[50,110,130],"(such":[51],"as":[52,116,126,128],"entities":[53,125],"and":[54,60,120,138,152,164],"relationships":[55,122],"text,":[57],"objects":[59,137],"object":[61],"attributes":[62],"images),":[64],"model's":[66,91,145],"capabilities":[67],"limited.":[69],"In":[70],"this":[71],"paper,":[72],"we":[73,148],"propose":[74],"advancing":[76],"Multi-modal":[77],"Structured":[78],"Knowledge":[79],"Representation":[80],"with":[81],"synergistic":[82],"hard":[83],"negative":[84,155],"samples":[85],"(MSKR),":[86],"thereby":[87],"significantly":[88,170],"improving":[89],"capability":[93],"for":[94],"such":[95,115],"data.":[96],"Specifically,":[97],"our":[98,195],"model":[99,196],"comprises":[100],"knowledge-enhanced":[103],"encoder":[104],"designed":[105],"to":[106,186],"bolster":[107],"inherent":[111],"textual":[113],"data,":[114],"entities,":[117],"attributes,":[119],"among":[123],"these":[124],"well":[127],"within":[131],"images,":[132],"focusing":[133],"on":[134,160,179],"elements":[135],"like":[136],"attributes.":[140],"To":[141],"further":[142],"refine":[143],"learning":[146,184,203],"process,":[147],"produce":[149],"both":[150],"image":[151],"text":[153],"challenging":[154],"samples.":[156],"Extensive":[157],"experimental":[158],"evaluations":[159],"Winoground,":[162],"InpaintCOCO,":[163],"MSCOCO":[165],"benchmark":[166],"reveal":[167],"MSKR":[169],"outperforms":[171],"baseline":[173],"model,":[174],"showcasing":[175],"marked":[176],"improvements":[177],"2.66%":[178],"average":[180],"representation":[183,191,202,211],"compared":[185],"baseline.":[188],"Moreover,":[189],"general":[190,210],"results":[192],"illustrate":[193],"not":[197],"only":[198],"excels":[199],"also":[205],"maintains":[206],"its":[207],"proficiency":[208],"learning.":[212]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":1}],"updated_date":"2026-04-10T15:06:20.359241","created_date":"2025-10-10T00:00:00"}