{"id":"https://openalex.org/W2952314184","doi":"https://doi.org/10.1145/3323873.3325049","title":"Understanding, Categorizing and Predicting Semantic Image-Text Relations","display_name":"Understanding, Categorizing and Predicting Semantic Image-Text Relations","publication_year":2019,"publication_date":"2019-06-05","ids":{"openalex":"https://openalex.org/W2952314184","doi":"https://doi.org/10.1145/3323873.3325049","mag":"2952314184"},"language":"en","primary_location":{"id":"doi:10.1145/3323873.3325049","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3323873.3325049","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3323873.3325049","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2019 on International Conference on Multimedia Retrieval","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3323873.3325049","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Christian Otto","orcid":null},"institutions":[{"id":"https://openalex.org/I2802635041","display_name":"Technische Informationsbibliothek (TIB)","ror":"https://ror.org/04aj4c181","country_code":"DE","type":"archive","lineage":["https://openalex.org/I2802635041","https://openalex.org/I315704651"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Christian Otto","raw_affiliation_strings":["Leibniz Information Centre for Science and Technology (TIB), Hannover, Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Leibniz Information Centre for Science and Technology (TIB), Hannover, Germany","institution_ids":["https://openalex.org/I2802635041"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Matthias Springstein","orcid":null},"institutions":[{"id":"https://openalex.org/I2802635041","display_name":"Technische Informationsbibliothek (TIB)","ror":"https://ror.org/04aj4c181","country_code":"DE","type":"archive","lineage":["https://openalex.org/I2802635041","https://openalex.org/I315704651"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Matthias Springstein","raw_affiliation_strings":["Leibniz Information Centre for Science and Technology (TIB), Hannover, Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Leibniz Information Centre for Science and Technology (TIB), Hannover, Germany","institution_ids":["https://openalex.org/I2802635041"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Avishek Anand","orcid":null},"institutions":[{"id":"https://openalex.org/I114112103","display_name":"Leibniz University Hannover","ror":"https://ror.org/0304hq317","country_code":"DE","type":"education","lineage":["https://openalex.org/I114112103"]},{"id":"https://openalex.org/I4210136150","display_name":"L3S Research Center","ror":"https://ror.org/039t4wk02","country_code":"DE","type":"facility","lineage":["https://openalex.org/I114112103","https://openalex.org/I4210136150","https://openalex.org/I94509681"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Avishek Anand","raw_affiliation_strings":["L3S Research Center, Leibniz Universit\u00e4t Hannover, Hannover, Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"L3S Research Center, Leibniz Universit\u00e4t Hannover, Hannover, Germany","institution_ids":["https://openalex.org/I114112103","https://openalex.org/I4210136150"]}]},{"author_position":"last","author":{"id":null,"display_name":"Ralph Ewerth","orcid":null},"institutions":[{"id":"https://openalex.org/I114112103","display_name":"Leibniz University Hannover","ror":"https://ror.org/0304hq317","country_code":"DE","type":"education","lineage":["https://openalex.org/I114112103"]},{"id":"https://openalex.org/I2802635041","display_name":"Technische Informationsbibliothek (TIB)","ror":"https://ror.org/04aj4c181","country_code":"DE","type":"archive","lineage":["https://openalex.org/I2802635041","https://openalex.org/I315704651"]},{"id":"https://openalex.org/I4210136150","display_name":"L3S Research Center","ror":"https://ror.org/039t4wk02","country_code":"DE","type":"facility","lineage":["https://openalex.org/I114112103","https://openalex.org/I4210136150","https://openalex.org/I94509681"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Ralph Ewerth","raw_affiliation_strings":["Leibniz Information Centre for Science and Technology (TIB) & L3S Research Center, Leibniz Universit\u00e4t Hannover, Hannover, Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Leibniz Information Centre for Science and Technology (TIB) & L3S Research Center, Leibniz Universit\u00e4t Hannover, Hannover, Germany","institution_ids":["https://openalex.org/I114112103","https://openalex.org/I2802635041","https://openalex.org/I4210136150"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.7322,"has_fulltext":false,"cited_by_count":18,"citation_normalized_percentile":{"value":0.79243982,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"168","last_page":"176"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10627","display_name":"Advanced Image and Video Retrieval Techniques","score":0.9955000281333923,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10824","display_name":"Image Retrieval and Classification Techniques","score":0.9850999712944031,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/categorization","display_name":"Categorization","score":0.6471999883651733},{"id":"https://openalex.org/keywords/closed-captioning","display_name":"Closed captioning","score":0.5758000016212463},{"id":"https://openalex.org/keywords/focus","display_name":"Focus (optics)","score":0.5547000169754028},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.519599974155426},{"id":"https://openalex.org/keywords/semantic-web","display_name":"Semantic Web","score":0.462799996137619},{"id":"https://openalex.org/keywords/variety","display_name":"Variety (cybernetics)","score":0.44760000705718994},{"id":"https://openalex.org/keywords/multimodal-learning","display_name":"Multimodal learning","score":0.44110000133514404},{"id":"https://openalex.org/keywords/modalities","display_name":"Modalities","score":0.4408999979496002},{"id":"https://openalex.org/keywords/relation","display_name":"Relation (database)","score":0.40709999203681946}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.794700026512146},{"id":"https://openalex.org/C94124525","wikidata":"https://www.wikidata.org/wiki/Q912550","display_name":"Categorization","level":2,"score":0.6471999883651733},{"id":"https://openalex.org/C157657479","wikidata":"https://www.wikidata.org/wiki/Q2367247","display_name":"Closed captioning","level":3,"score":0.5758000016212463},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.5547000169754028},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5490999817848206},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.5281000137329102},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.519599974155426},{"id":"https://openalex.org/C2129575","wikidata":"https://www.wikidata.org/wiki/Q54837","display_name":"Semantic Web","level":2,"score":0.462799996137619},{"id":"https://openalex.org/C136197465","wikidata":"https://www.wikidata.org/wiki/Q1729295","display_name":"Variety (cybernetics)","level":2,"score":0.44760000705718994},{"id":"https://openalex.org/C2780660688","wikidata":"https://www.wikidata.org/wiki/Q25052564","display_name":"Multimodal learning","level":2,"score":0.44110000133514404},{"id":"https://openalex.org/C2779903281","wikidata":"https://www.wikidata.org/wiki/Q6888026","display_name":"Modalities","level":2,"score":0.4408999979496002},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.4223000109195709},{"id":"https://openalex.org/C25343380","wikidata":"https://www.wikidata.org/wiki/Q277521","display_name":"Relation (database)","level":2,"score":0.40709999203681946},{"id":"https://openalex.org/C184337299","wikidata":"https://www.wikidata.org/wiki/Q1437428","display_name":"Semantics (computer science)","level":2,"score":0.40529999136924744},{"id":"https://openalex.org/C130318100","wikidata":"https://www.wikidata.org/wiki/Q2268914","display_name":"Semantic similarity","level":2,"score":0.3959999978542328},{"id":"https://openalex.org/C36464697","wikidata":"https://www.wikidata.org/wiki/Q451553","display_name":"Visualization","level":2,"score":0.373199999332428},{"id":"https://openalex.org/C511149849","wikidata":"https://www.wikidata.org/wiki/Q7449051","display_name":"Semantic computing","level":3,"score":0.35850000381469727},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.34619998931884766},{"id":"https://openalex.org/C534406577","wikidata":"https://www.wikidata.org/wiki/Q7550843","display_name":"Social Semantic Web","level":3,"score":0.33070001006126404},{"id":"https://openalex.org/C2780226545","wikidata":"https://www.wikidata.org/wiki/Q6888030","display_name":"Modality (human\u2013computer interaction)","level":2,"score":0.3061000108718872},{"id":"https://openalex.org/C2988080768","wikidata":"https://www.wikidata.org/wiki/Q7095057","display_name":"Semantic relation","level":3,"score":0.29809999465942383},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.29660001397132874},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.29490000009536743},{"id":"https://openalex.org/C58489278","wikidata":"https://www.wikidata.org/wiki/Q1172284","display_name":"Data set","level":2,"score":0.2833999991416931},{"id":"https://openalex.org/C90312973","wikidata":"https://www.wikidata.org/wiki/Q7449052","display_name":"Semantic data model","level":2,"score":0.2793000042438507},{"id":"https://openalex.org/C557471498","wikidata":"https://www.wikidata.org/wiki/Q554950","display_name":"Recommender system","level":2,"score":0.26339998841285706}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3323873.3325049","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3323873.3325049","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3323873.3325049","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2019 on International Conference on Multimedia Retrieval","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:1906.08595","is_oa":true,"landing_page_url":"http://arxiv.org/abs/1906.08595","pdf_url":"https://arxiv.org/pdf/1906.08595","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"}],"best_oa_location":{"id":"doi:10.1145/3323873.3325049","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3323873.3325049","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3323873.3325049","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2019 on International Conference on Multimedia Retrieval","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":27,"referenced_works":["https://openalex.org/W1977547683","https://openalex.org/W2019024625","https://openalex.org/W2097117768","https://openalex.org/W2112019442","https://openalex.org/W2117539524","https://openalex.org/W2130660124","https://openalex.org/W2194775991","https://openalex.org/W2251394420","https://openalex.org/W2337005149","https://openalex.org/W2470673105","https://openalex.org/W2489370933","https://openalex.org/W2523993696","https://openalex.org/W2526093969","https://openalex.org/W2618264341","https://openalex.org/W2745461083","https://openalex.org/W2749708282","https://openalex.org/W2767193869","https://openalex.org/W2767484504","https://openalex.org/W2808336545","https://openalex.org/W2808399042","https://openalex.org/W2888166343","https://openalex.org/W2963446712","https://openalex.org/W2963758027","https://openalex.org/W2964350391","https://openalex.org/W3004387475","https://openalex.org/W4247784431","https://openalex.org/W6811886535"],"related_works":[],"abstract_inverted_index":{"Two":[0],"modalities":[1],"are":[2],"often":[3],"used":[4],"to":[5,79,168],"convey":[6],"information":[7,57,119],"in":[8,15,110],"a":[9,41,123,143,164,178,193,210],"complementary":[10],"and":[11,32,49,76,105,134,153,159,189,197],"beneficial":[12],"manner,":[13],"e.g.,":[14],"online":[16],"news,":[17],"videos,":[18],"educational":[19],"resources,":[20,199],"or":[21,91,132],"scientific":[22],"publications.":[23],"The":[24],"automatic":[25,53],"understanding":[26,54],"of":[27,55,93,125,145,157,182,195,217],"semantic":[28,85,114,127,151],"correlations":[29],"between":[30],"text":[31],"associated":[33],"images":[34],"as":[35,37,67],"well":[36],"their":[38],"interplay":[39],"has":[40],"great":[42],"potential":[43],"for":[44,117],"enhanced":[45],"multimodal":[46,56,118,174],"web":[47,198],"search":[48],"recommender":[50],"systems.":[51],"However,":[52],"is":[58],"still":[59],"an":[60,94],"unsolved":[61],"research":[62,109,203],"problem.":[63],"Recent":[64],"approaches":[65],"such":[66],"image":[68,158],"captioning":[69],"focus":[70],"on":[71,204,209],"precisely":[72],"describing":[73],"visual":[74,111],"content":[75],"translating":[77],"it":[78],"text,":[80],"but":[81],"typically":[82],"address":[83],"neither":[84],"interpretations":[86],"nor":[87],"the":[88,154,215,218],"specific":[89],"role":[90],"purpose":[92],"image-text":[95,115,128],"constellation.":[96],"In":[97],"this":[98,205],"paper,":[99],"we":[100,162,185],"go":[101],"beyond":[102],"previous":[103],"work":[104],"investigate,":[106],"inspired":[107],"by":[108,142,172],"communication,":[112],"useful":[113],"relations":[116],"retrieval.":[120],"We":[121],"derive":[122],"categorization":[124],"eight":[126],"classes":[129,171],"(e.g.,":[130],"\"illustration\"":[131],"\"anchorage\")":[133],"show":[135],"how":[136],"they":[137],"can":[138],"systematically":[139],"be":[140],"characterized":[141],"set":[144,213],"three":[146],"metrics:":[147],"cross-modal":[148],"mutual":[149],"information,":[150],"correlation,":[152],"status":[155],"relation":[156],"text.":[160],"Furthermore,":[161],"present":[163],"deep":[165],"learning":[166],"system":[167],"predict":[169],"these":[170],"utilizing":[173],"embeddings.":[175],"To":[176],"obtain":[177],"sufficiently":[179],"large":[180],"amount":[181],"training":[183],"data,":[184],"have":[186],"automatically":[187],"collected":[188],"augmented":[190],"data":[191],"from":[192],"variety":[194],"datasets":[196],"which":[200],"enables":[201],"future":[202],"topic.":[206],"Experimental":[207],"results":[208],"demanding":[211],"test":[212],"demonstrate":[214],"feasibility":[216],"approach.":[219]},"counts_by_year":[{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":3},{"year":2023,"cited_by_count":5},{"year":2022,"cited_by_count":2},{"year":2021,"cited_by_count":5},{"year":2020,"cited_by_count":1}],"updated_date":"2026-08-05T07:39:15.569665","created_date":"2019-06-27T00:00:00"}