{"id":"https://openalex.org/W3048782180","doi":"https://doi.org/10.1145/3451390","title":"Fine-Grained Visual Textual Alignment for Cross-Modal Retrieval Using Transformer Encoders","display_name":"Fine-Grained Visual Textual Alignment for Cross-Modal Retrieval Using Transformer Encoders","publication_year":2021,"publication_date":"2021-11-12","ids":{"openalex":"https://openalex.org/W3048782180","doi":"https://doi.org/10.1145/3451390","mag":"3048782180"},"language":"en","primary_location":{"id":"doi:10.1145/3451390","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3451390","pdf_url":null,"source":{"id":"https://openalex.org/S19610489","display_name":"ACM Transactions on Multimedia Computing Communications and Applications","issn_l":"1551-6857","issn":["1551-6857","1551-6865"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Multimedia Computing, Communications, and Applications","raw_type":"journal-article"},"type":"article","indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2008.05231","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5007125882","display_name":"Nicola Messina","orcid":"https://orcid.org/0000-0003-3011-2487"},"institutions":[{"id":"https://openalex.org/I122991210","display_name":"Istituto di Scienza e Tecnologie dell'Informazione \"Alessandro Faedo\"","ror":"https://ror.org/05kacka20","country_code":"IT","type":"facility","lineage":["https://openalex.org/I122991210","https://openalex.org/I4210155236"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Nicola Messina","raw_affiliation_strings":["ISTI-CNR, Pisa, Italy","ISTI-CNR Pisa Italy#TAB#"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ISTI-CNR, Pisa, Italy","institution_ids":["https://openalex.org/I122991210"]},{"raw_affiliation_string":"ISTI-CNR Pisa Italy#TAB#","institution_ids":["https://openalex.org/I122991210"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5072396889","display_name":"Giuseppe Amato","orcid":"https://orcid.org/0000-0003-0171-4315"},"institutions":[{"id":"https://openalex.org/I122991210","display_name":"Istituto di Scienza e Tecnologie dell'Informazione \"Alessandro Faedo\"","ror":"https://ror.org/05kacka20","country_code":"IT","type":"facility","lineage":["https://openalex.org/I122991210","https://openalex.org/I4210155236"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Giuseppe Amato","raw_affiliation_strings":["ISTI-CNR, Pisa, Italy","ISTI-CNR Pisa Italy#TAB#"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ISTI-CNR, Pisa, Italy","institution_ids":["https://openalex.org/I122991210"]},{"raw_affiliation_string":"ISTI-CNR Pisa Italy#TAB#","institution_ids":["https://openalex.org/I122991210"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5082069715","display_name":"Andrea Esuli","orcid":"https://orcid.org/0000-0002-5725-4322"},"institutions":[{"id":"https://openalex.org/I122991210","display_name":"Istituto di Scienza e Tecnologie dell'Informazione \"Alessandro Faedo\"","ror":"https://ror.org/05kacka20","country_code":"IT","type":"facility","lineage":["https://openalex.org/I122991210","https://openalex.org/I4210155236"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Andrea Esuli","raw_affiliation_strings":["ISTI-CNR, Pisa, Italy","ISTI-CNR Pisa Italy#TAB#"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ISTI-CNR, Pisa, Italy","institution_ids":["https://openalex.org/I122991210"]},{"raw_affiliation_string":"ISTI-CNR Pisa Italy#TAB#","institution_ids":["https://openalex.org/I122991210"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5047189288","display_name":"Fabrizio Falchi","orcid":"https://orcid.org/0000-0001-6258-5313"},"institutions":[{"id":"https://openalex.org/I122991210","display_name":"Istituto di Scienza e Tecnologie dell'Informazione \"Alessandro Faedo\"","ror":"https://ror.org/05kacka20","country_code":"IT","type":"facility","lineage":["https://openalex.org/I122991210","https://openalex.org/I4210155236"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Fabrizio Falchi","raw_affiliation_strings":["ISTI-CNR, Pisa, Italy","ISTI-CNR Pisa Italy#TAB#"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ISTI-CNR, Pisa, Italy","institution_ids":["https://openalex.org/I122991210"]},{"raw_affiliation_string":"ISTI-CNR Pisa Italy#TAB#","institution_ids":["https://openalex.org/I122991210"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5072879702","display_name":"Claudio Gennaro","orcid":"https://orcid.org/0000-0002-3715-149X"},"institutions":[{"id":"https://openalex.org/I122991210","display_name":"Istituto di Scienza e Tecnologie dell'Informazione \"Alessandro Faedo\"","ror":"https://ror.org/05kacka20","country_code":"IT","type":"facility","lineage":["https://openalex.org/I122991210","https://openalex.org/I4210155236"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Claudio Gennaro","raw_affiliation_strings":["ISTI-CNR, Pisa, Italy","ISTI-CNR Pisa Italy#TAB#"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ISTI-CNR, Pisa, Italy","institution_ids":["https://openalex.org/I122991210"]},{"raw_affiliation_string":"ISTI-CNR Pisa Italy#TAB#","institution_ids":["https://openalex.org/I122991210"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5061688822","display_name":"St\u00e9phane Marchand\u2010Maillet","orcid":"https://orcid.org/0000-0002-4875-6101"},"institutions":[{"id":"https://openalex.org/I114457229","display_name":"University of Geneva","ror":"https://ror.org/01swzsf04","country_code":"CH","type":"education","lineage":["https://openalex.org/I114457229"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"St\u00e9phane Marchand-Maillet","raw_affiliation_strings":["VIPER Group\u2013University of Geneva, Geneva, Switzerland","Viper group - University of Geneva, Geneva, Switzerland#TAB#"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"VIPER Group\u2013University of Geneva, Geneva, Switzerland","institution_ids":["https://openalex.org/I114457229"]},{"raw_affiliation_string":"Viper group - University of Geneva, Geneva, Switzerland#TAB#","institution_ids":["https://openalex.org/I114457229"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":1.4463,"has_fulltext":false,"cited_by_count":16,"citation_normalized_percentile":{"value":0.8137555,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":94,"max":99},"biblio":{"volume":"17","issue":"4","first_page":"1","last_page":"23"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10627","display_name":"Advanced Image and Video Retrieval Techniques","score":0.9987000226974487,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.9941999912261963,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7369614839553833},{"id":"https://openalex.org/keywords/sentence","display_name":"Sentence","score":0.639061450958252},{"id":"https://openalex.org/keywords/question-answering","display_name":"Question answering","score":0.5548989772796631},{"id":"https://openalex.org/keywords/modal","display_name":"Modal","score":0.5212400555610657},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.49948644638061523},{"id":"https://openalex.org/keywords/search-engine-indexing","display_name":"Search engine indexing","score":0.49628549814224243},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.48211002349853516},{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.47293806076049805},{"id":"https://openalex.org/keywords/encoder","display_name":"Encoder","score":0.46587061882019043},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.382485568523407}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7369614839553833},{"id":"https://openalex.org/C2777530160","wikidata":"https://www.wikidata.org/wiki/Q41796","display_name":"Sentence","level":2,"score":0.639061450958252},{"id":"https://openalex.org/C44291984","wikidata":"https://www.wikidata.org/wiki/Q1074173","display_name":"Question answering","level":2,"score":0.5548989772796631},{"id":"https://openalex.org/C71139939","wikidata":"https://www.wikidata.org/wiki/Q910194","display_name":"Modal","level":2,"score":0.5212400555610657},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.49948644638061523},{"id":"https://openalex.org/C75165309","wikidata":"https://www.wikidata.org/wiki/Q2258979","display_name":"Search engine indexing","level":2,"score":0.49628549814224243},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.48211002349853516},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.47293806076049805},{"id":"https://openalex.org/C118505674","wikidata":"https://www.wikidata.org/wiki/Q42586063","display_name":"Encoder","level":2,"score":0.46587061882019043},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.382485568523407},{"id":"https://openalex.org/C188027245","wikidata":"https://www.wikidata.org/wiki/Q750446","display_name":"Polymer chemistry","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.0}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.1145/3451390","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3451390","pdf_url":null,"source":{"id":"https://openalex.org/S19610489","display_name":"ACM Transactions on Multimedia Computing Communications and Applications","issn_l":"1551-6857","issn":["1551-6857","1551-6865"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Multimedia Computing, Communications, and Applications","raw_type":"journal-article"},{"id":"pmh:oai:arXiv.org:2008.05231","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2008.05231","pdf_url":"https://arxiv.org/pdf/2008.05231","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"mag:3048782180","is_oa":true,"landing_page_url":"https://arxiv.org/pdf/2008.05231.pdf","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null},{"id":"doi:10.48550/arxiv.2008.05231","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2008.05231","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2008.05231","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2008.05231","pdf_url":"https://arxiv.org/pdf/2008.05231","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[{"score":0.6899999976158142,"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":71,"referenced_works":["https://openalex.org/W1614298861","https://openalex.org/W1861492603","https://openalex.org/W1905882502","https://openalex.org/W1957706851","https://openalex.org/W2185175083","https://openalex.org/W2277195237","https://openalex.org/W2346425926","https://openalex.org/W2467557055","https://openalex.org/W2506483933","https://openalex.org/W2552579943","https://openalex.org/W2624614404","https://openalex.org/W2745461083","https://openalex.org/W2778100917","https://openalex.org/W2810482788","https://openalex.org/W2886970679","https://openalex.org/W2890531016","https://openalex.org/W2896457183","https://openalex.org/W2896798564","https://openalex.org/W2900953995","https://openalex.org/W2901988662","https://openalex.org/W2903529742","https://openalex.org/W2913618459","https://openalex.org/W2942614241","https://openalex.org/W2953106684","https://openalex.org/W2962716332","https://openalex.org/W2962779575","https://openalex.org/W2962964995","https://openalex.org/W2963040148","https://openalex.org/W2963084599","https://openalex.org/W2963101956","https://openalex.org/W2963224792","https://openalex.org/W2963403868","https://openalex.org/W2963467339","https://openalex.org/W2963499204","https://openalex.org/W2963677766","https://openalex.org/W2963899908","https://openalex.org/W2964120214","https://openalex.org/W2965848243","https://openalex.org/W2970608575","https://openalex.org/W2972895044","https://openalex.org/W2973978812","https://openalex.org/W2975501350","https://openalex.org/W2979304729","https://openalex.org/W2981586349","https://openalex.org/W2982078236","https://openalex.org/W2984814708","https://openalex.org/W2985951359","https://openalex.org/W2986670728","https://openalex.org/W2988823324","https://openalex.org/W2989176720","https://openalex.org/W2990571534","https://openalex.org/W2995460200","https://openalex.org/W2997591391","https://openalex.org/W3001555892","https://openalex.org/W3005971801","https://openalex.org/W3008356057","https://openalex.org/W3014611590","https://openalex.org/W3017098848","https://openalex.org/W3020257313","https://openalex.org/W3021244424","https://openalex.org/W3028452400","https://openalex.org/W3035454331","https://openalex.org/W3035588244","https://openalex.org/W3035605030","https://openalex.org/W3048588868","https://openalex.org/W3090449556","https://openalex.org/W3092820619","https://openalex.org/W3104180728","https://openalex.org/W3105817677","https://openalex.org/W3160179442","https://openalex.org/W6767769535"],"related_works":["https://openalex.org/W3170097241","https://openalex.org/W3090449556","https://openalex.org/W2962964995","https://openalex.org/W3025923720","https://openalex.org/W3062155679","https://openalex.org/W2783179304","https://openalex.org/W2994818707","https://openalex.org/W2970608575","https://openalex.org/W2963341956","https://openalex.org/W2185175083","https://openalex.org/W1538243752","https://openalex.org/W3206569427","https://openalex.org/W170198589","https://openalex.org/W2299727453","https://openalex.org/W1978962787","https://openalex.org/W3203247393","https://openalex.org/W2140331783","https://openalex.org/W2994974226","https://openalex.org/W2949436286","https://openalex.org/W3160926710"],"abstract_inverted_index":{"Despite":[0],"the":[1,20,36,60,75,86,105,120,142,146,159,162,167,173,179,186,189,203,213,228,231,236,243],"evolution":[2],"of":[3,22,63,78,205,222],"deep-learning-based":[4],"visual-textual":[5],"processing":[6],"systems,":[7],"precise":[8],"multi-modal":[9],"matching":[10,27],"remains":[11],"a":[12,43,56],"challenging":[13],"task.":[14,108],"In":[15,154],"this":[16,155],"work,":[17],"we":[18,41,218],"tackle":[19],"task":[21,89],"cross-modal":[23,112,198],"retrieval":[24,88,107,152,233],"through":[25],"image-sentence":[26,38],"based":[28],"on":[29,85,90,97,104,110,227,235,248],"word-region":[30],"alignments,":[31],"using":[32],"supervision":[33],"only":[34,165],"at":[35,250],"global":[37],"level.":[39],"Specifically,":[40],"present":[42],"novel":[44],"approach":[45,207],"called":[46],"Transformer":[47],"Encoder":[48],"Reasoning":[49],"and":[50,65,70,93,122,137,145,193,224,230],"Alignment":[51],"Network":[52],"(TERAN).":[53],"TERAN":[54,81,115,157,184],"enforces":[55],"fine-grained":[57,180],"match":[58],"between":[59],"underlying":[61],"components":[62],"images":[64],"sentences":[66],"(i.e.,":[67],"image":[68,87,229],"regions":[69],"words,":[71],"respectively)":[72],"to":[73,118,133],"preserve":[74],"informative":[76],"richness":[77],"both":[79,91],"modalities.":[80],"obtains":[82],"state-of-the-art":[83,210],"results":[84],"MS-COCO":[92,214],"Flickr30k":[94],"datasets.":[95],"Moreover,":[96],"MS-COCO,":[98],"it":[99],"also":[100],"outperforms":[101],"current":[102],"approaches":[103],"sentence":[106,232],"Focusing":[109],"scalable":[111],"information":[113,160,199],"retrieval,":[114],"is":[116,245],"designed":[117],"keep":[119],"visual":[121,136],"textual":[123,138],"data":[124],"pipelines":[125],"well":[126],"separated.":[127],"Cross-attention":[128],"links":[129],"invalidate":[130],"any":[131],"chance":[132],"separately":[134],"extract":[135],"features":[139],"needed":[140],"for":[141,191,196,242],"online":[143],"search":[144],"offline":[147],"indexing":[148],"steps":[149],"in":[150],"large-scale":[151,197],"systems.":[153],"respect,":[156],"merges":[158],"from":[161],"two":[163],"domains":[164],"during":[166],"final":[168],"alignment":[169],"phase,":[170],"immediately":[171],"before":[172],"loss":[174],"computation.":[175],"We":[176,201],"argue":[177],"that":[178],"alignments":[181],"produced":[182],"by":[183],"pave":[185],"way":[187],"toward":[188],"research":[190],"effective":[192],"efficient":[194],"methods":[195],"retrieval.":[200],"compare":[202],"effectiveness":[204],"our":[206],"against":[208],"relevant":[209],"methods.":[211],"On":[212],"1K":[215],"test":[216],"set,":[217],"obtain":[219],"an":[220],"improvement":[221],"5.7%":[223],"3.5%":[225],"respectively":[226],"tasks":[234],"Recall@1":[237],"metric.":[238],"The":[239],"code":[240],"used":[241],"experiments":[244],"publicly":[246],"available":[247],"GitHub":[249],"https://github.com/mesnico/TERAN":[251],".":[252]},"counts_by_year":[{"year":2024,"cited_by_count":4},{"year":2023,"cited_by_count":2},{"year":2022,"cited_by_count":2},{"year":2021,"cited_by_count":8}],"updated_date":"2026-07-15T18:14:33.161393","created_date":"2022-07-26T00:00:00"}