{"id":"https://openalex.org/W4377121434","doi":"https://doi.org/10.48550/arxiv.2305.10400","title":"What You See is What You Read? Improving Text-Image Alignment Evaluation","display_name":"What You See is What You Read? Improving Text-Image Alignment Evaluation","publication_year":2023,"publication_date":"2023-05-17","ids":{"openalex":"https://openalex.org/W4377121434","doi":"https://doi.org/10.48550/arxiv.2305.10400"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2305.10400","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2305.10400","pdf_url":"https://arxiv.org/pdf/2305.10400","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2305.10400","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5034783276","display_name":"Michal Yarom","orcid":"https://orcid.org/0000-0002-2330-0378"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yarom, Michal","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5068580969","display_name":"Yonatan Bitton","orcid":"https://orcid.org/0000-0002-1185-6838"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Bitton, Yonatan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5075518871","display_name":"Soravit Changpinyo","orcid":"https://orcid.org/0000-0002-4013-1190"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Changpinyo, Soravit","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5027223654","display_name":"Roee Aharoni","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Aharoni, Roee","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5071893787","display_name":"Jonathan Herzig","orcid":"https://orcid.org/0009-0000-7227-6557"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Herzig, Jonathan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5035078692","display_name":"Oran Lang","orcid":"https://orcid.org/0000-0001-7644-8459"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lang, Oran","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5078910984","display_name":"E. O. Ofek","orcid":"https://orcid.org/0000-0002-6786-8774"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ofek, Eran","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5026091724","display_name":"Idan Szpektor","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Szpektor, Idan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":14,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9878000020980835,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.980400025844574,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8051720857620239},{"id":"https://openalex.org/keywords/image","display_name":"Image (mathematics)","score":0.6927599310874939},{"id":"https://openalex.org/keywords/pipeline","display_name":"Pipeline (software)","score":0.672462522983551},{"id":"https://openalex.org/keywords/generative-grammar","display_name":"Generative grammar","score":0.630967378616333},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.6050349473953247},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5877771973609924},{"id":"https://openalex.org/keywords/rank","display_name":"Rank (graph theory)","score":0.5805510878562927},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.4495115876197815},{"id":"https://openalex.org/keywords/generative-model","display_name":"Generative model","score":0.411314994096756},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.4099617898464203},{"id":"https://openalex.org/keywords/pattern-recognition","display_name":"Pattern recognition (psychology)","score":0.39266088604927063},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.06648227572441101}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8051720857620239},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.6927599310874939},{"id":"https://openalex.org/C43521106","wikidata":"https://www.wikidata.org/wiki/Q2165493","display_name":"Pipeline (software)","level":2,"score":0.672462522983551},{"id":"https://openalex.org/C39890363","wikidata":"https://www.wikidata.org/wiki/Q36108","display_name":"Generative grammar","level":2,"score":0.630967378616333},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.6050349473953247},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5877771973609924},{"id":"https://openalex.org/C164226766","wikidata":"https://www.wikidata.org/wiki/Q7293202","display_name":"Rank (graph theory)","level":2,"score":0.5805510878562927},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.4495115876197815},{"id":"https://openalex.org/C167966045","wikidata":"https://www.wikidata.org/wiki/Q5532625","display_name":"Generative model","level":3,"score":0.411314994096756},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.4099617898464203},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.39266088604927063},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.06648227572441101},{"id":"https://openalex.org/C114614502","wikidata":"https://www.wikidata.org/wiki/Q76592","display_name":"Combinatorics","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2305.10400","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2305.10400","pdf_url":"https://arxiv.org/pdf/2305.10400","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2305.10400","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2305.10400","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2305.10400","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2305.10400","pdf_url":"https://arxiv.org/pdf/2305.10400","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4377121434.pdf","grobid_xml":"https://content.openalex.org/works/W4377121434.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4365211920","https://openalex.org/W3014948380","https://openalex.org/W4380551139","https://openalex.org/W4317695495","https://openalex.org/W4387506531","https://openalex.org/W4238433571","https://openalex.org/W3174044702","https://openalex.org/W2967848559","https://openalex.org/W4299831724","https://openalex.org/W4283803360"],"abstract_inverted_index":{"Automatically":[0],"determining":[1],"whether":[2,60],"a":[3,6,13,42,61,80,141],"text":[4],"and":[5,24,52,86,91,140,144],"corresponding":[7],"image":[8,139],"are":[9],"semantically":[10,66],"aligned":[11],"is":[12,65],"significant":[14,115],"challenge":[15],"for":[16,33,59],"vision-language":[17],"models,":[18,90],"with":[19,56,114],"applications":[20],"in":[21,109,117,154],"generative":[22],"text-to-image":[23,51,155],"image-to-text":[25,53],"tasks.":[26],"In":[27],"this":[28],"work,":[29],"we":[30,128],"study":[31],"methods":[32,73,105],"automatic":[34,72],"text-image":[35,63,111],"alignment":[36,112],"evaluation.":[37],"We":[38,68],"first":[39,78],"introduce":[40],"SeeTRUE:":[41],"comprehensive":[43],"evaluation":[44],"set,":[45],"spanning":[46],"multiple":[47],"datasets":[48],"from":[49],"both":[50],"generation":[54,85],"tasks,":[55,113],"human":[57],"judgements":[58],"given":[62,142],"pair":[64],"aligned.":[67],"then":[69],"describe":[70],"two":[71],"to":[74,150],"determine":[75],"alignment:":[76],"the":[77,92],"involving":[79],"pipeline":[81],"based":[82],"on":[83],"question":[84,88],"visual":[87],"answering":[89],"second":[93],"employing":[94],"an":[95,138],"end-to-end":[96],"classification":[97],"approach":[98],"by":[99],"finetuning":[100],"multimodal":[101],"pretrained":[102],"models.":[103],"Both":[104],"surpass":[106],"prior":[107],"approaches":[108,132],"various":[110],"improvements":[116],"challenging":[118],"cases":[119],"that":[120],"involve":[121],"complex":[122],"composition":[123],"or":[124],"unnatural":[125],"images.":[126],"Finally,":[127],"demonstrate":[129],"how":[130,145],"our":[131],"can":[133,147],"localize":[134],"specific":[135],"misalignments":[136],"between":[137],"text,":[143],"they":[146],"be":[148],"used":[149],"automatically":[151],"re-rank":[152],"candidates":[153],"generation.":[156]},"counts_by_year":[{"year":2025,"cited_by_count":4},{"year":2024,"cited_by_count":8},{"year":2023,"cited_by_count":2}],"updated_date":"2026-08-11T07:18:39.950985","created_date":"2025-10-10T00:00:00"}