{"id":"https://openalex.org/W4388327564","doi":"https://doi.org/10.48550/arxiv.2311.01361","title":"GPT-4V(ision) as a Generalist Evaluator for Vision-Language Tasks","display_name":"GPT-4V(ision) as a Generalist Evaluator for Vision-Language Tasks","alternate_titles":[],"publication_year":2023,"publication_date":"2023-11-02","ids":{"openalex":"https://openalex.org/W4388327564","doi":"https://doi.org/10.48550/arxiv.2311.01361"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2311.01361","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2311.01361","pdf_url":"https://arxiv.org/pdf/2311.01361","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"parent_works":[],"lineage":["https://openalex.org/W4388327564"],"child_works_count":0,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2311.01361","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5102845705","display_name":"Xinlu Zhang","orcid":"https://orcid.org/0009-0002-5585-062X","observed_orcids":["https://orcid.org/0009-0002-5585-062X","https://orcid.org/0009-0001-2291-3914"]},"institutions":[{"id":"https://openalex.org/I1174212","display_name":"University of Southern California","ror":"https://ror.org/03taz7m60","country_code":"US","type":"education","lineage":["https://openalex.org/I1174212"]},{"id":"https://openalex.org/I4405256866","display_name":"ByteDance (China)","ror":"https://ror.org/04m8av638","country_code":"CN","type":"company","lineage":["https://openalex.org/I4405256866"]}],"countries":["CN","US"],"is_corresponding":false,"raw_author_name":"Zhang, Xinlu","raw_affiliation_strings":["\u2663 University of Southern California \u2661 Bytedance , US"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"\u2663 University of Southern California \u2661 Bytedance , US","institution_ids":["https://openalex.org/I1174212","https://openalex.org/I4405256866"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5036525093","display_name":"Yujie Lu","orcid":"https://orcid.org/0000-0002-0691-2129","observed_orcids":["https://orcid.org/0000-0002-0691-2129"]},"institutions":[{"id":"https://openalex.org/I1174212","display_name":"University of Southern California","ror":"https://ror.org/03taz7m60","country_code":"US","type":"education","lineage":["https://openalex.org/I1174212"]},{"id":"https://openalex.org/I4405256866","display_name":"ByteDance (China)","ror":"https://ror.org/04m8av638","country_code":"CN","type":"company","lineage":["https://openalex.org/I4405256866"]}],"countries":["CN","US"],"is_corresponding":false,"raw_author_name":"Lu, Yujie","raw_affiliation_strings":["\u2663 University of Southern California \u2661 Bytedance , US"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"\u2663 University of Southern California \u2661 Bytedance , US","institution_ids":["https://openalex.org/I1174212","https://openalex.org/I4405256866"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Wang, Weizhi","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I154570441","display_name":"University of California, Santa Barbara","ror":"https://ror.org/02t274463","country_code":"US","type":"education","lineage":["https://openalex.org/I154570441"]},{"id":"https://openalex.org/I36258959","display_name":"University of California San Diego","ror":"https://ror.org/0168r3w48","country_code":"US","type":"education","lineage":["https://openalex.org/I36258959"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Wang, Weizhi","raw_affiliation_strings":["University of California , Santa Barbara \u2660 University of California , San Diego"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of California , Santa Barbara \u2660 University of California , San Diego","institution_ids":["https://openalex.org/I154570441","https://openalex.org/I36258959"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Yan, An","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I154570441","display_name":"University of California, Santa Barbara","ror":"https://ror.org/02t274463","country_code":"US","type":"education","lineage":["https://openalex.org/I154570441"]},{"id":"https://openalex.org/I36258959","display_name":"University of California San Diego","ror":"https://ror.org/0168r3w48","country_code":"US","type":"education","lineage":["https://openalex.org/I36258959"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yan, An","raw_affiliation_strings":["University of California , Santa Barbara \u2660 University of California , San Diego"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of California , Santa Barbara \u2660 University of California , San Diego","institution_ids":["https://openalex.org/I154570441","https://openalex.org/I36258959"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100369186","display_name":"Jun Yan","orcid":"https://orcid.org/0009-0001-2415-5125","observed_orcids":["https://orcid.org/0009-0001-2415-5125"]},"institutions":[{"id":"https://openalex.org/I154570441","display_name":"University of California, Santa Barbara","ror":"https://ror.org/02t274463","country_code":"US","type":"education","lineage":["https://openalex.org/I154570441"]},{"id":"https://openalex.org/I36258959","display_name":"University of California San Diego","ror":"https://ror.org/0168r3w48","country_code":"US","type":"education","lineage":["https://openalex.org/I36258959"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yan, Jun","raw_affiliation_strings":["University of California , Santa Barbara \u2660 University of California , San Diego"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of California , Santa Barbara \u2660 University of California , San Diego","institution_ids":["https://openalex.org/I154570441","https://openalex.org/I36258959"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5048533277","display_name":"Lianke Qin","orcid":"https://orcid.org/0000-0002-1259-7137","observed_orcids":["https://orcid.org/0000-0002-1259-7137"]},"institutions":[{"id":"https://openalex.org/I154570441","display_name":"University of California, Santa Barbara","ror":"https://ror.org/02t274463","country_code":"US","type":"education","lineage":["https://openalex.org/I154570441"]},{"id":"https://openalex.org/I36258959","display_name":"University of California San Diego","ror":"https://ror.org/0168r3w48","country_code":"US","type":"education","lineage":["https://openalex.org/I36258959"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Qin, Lianke","raw_affiliation_strings":["University of California , Santa Barbara \u2660 University of California , San Diego"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of California , Santa Barbara \u2660 University of California , San Diego","institution_ids":["https://openalex.org/I154570441","https://openalex.org/I36258959"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100453993","display_name":"Heng Wang","orcid":"https://orcid.org/0009-0009-5473-5751","observed_orcids":["https://orcid.org/0009-0009-5473-5751"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Heng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5047709762","display_name":"Xifeng Yan","orcid":"https://orcid.org/0009-0000-6508-4792","observed_orcids":["https://orcid.org/0009-0000-6508-4792"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yan, Xifeng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100702485","display_name":"William Yang Wang","orcid":"https://orcid.org/0000-0001-6153-8240","observed_orcids":["https://orcid.org/0000-0001-6153-8240"]},"institutions":[{"id":"https://openalex.org/I154570441","display_name":"University of California, Santa Barbara","ror":"https://ror.org/02t274463","country_code":"US","type":"education","lineage":["https://openalex.org/I154570441"]},{"id":"https://openalex.org/I36258959","display_name":"University of California San Diego","ror":"https://ror.org/0168r3w48","country_code":"US","type":"education","lineage":["https://openalex.org/I36258959"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Wang, William Yang","raw_affiliation_strings":["University of California , Santa Barbara \u2660 University of California , San Diego"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of California , Santa Barbara \u2660 University of California , San Diego","institution_ids":["https://openalex.org/I154570441","https://openalex.org/I36258959"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5021640058","display_name":"Linda Ruth Petzold","orcid":"https://orcid.org/0000-0001-6251-6078","observed_orcids":["https://orcid.org/0000-0001-6251-6078"]},"institutions":[{"id":"https://openalex.org/I154570441","display_name":"University of California, Santa Barbara","ror":"https://ror.org/02t274463","country_code":"US","type":"education","lineage":["https://openalex.org/I154570441"]},{"id":"https://openalex.org/I36258959","display_name":"University of California San Diego","ror":"https://ror.org/0168r3w48","country_code":"US","type":"education","lineage":["https://openalex.org/I36258959"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Petzold, Linda Ruth","raw_affiliation_strings":["University of California , Santa Barbara \u2660 University of California , San Diego"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of California , Santa Barbara \u2660 University of California , San Diego","institution_ids":["https://openalex.org/I154570441","https://openalex.org/I36258959"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":168,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_pulled":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10775","display_name":"Generative Adversarial Networks and Image Synthesis","score":9.999999747378752e-05,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/gpt-4v","display_name":"GPT-4V","score":0.9950000047683716},{"id":"https://openalex.org/keywords/automated-assessment","display_name":"automated assessment","score":0.7739999890327454},{"id":"https://openalex.org/keywords/vision-language-tasks","display_name":"vision-language tasks","score":0.8029999732971191},{"id":"https://openalex.org/keywords/multimodal-large-language-models","display_name":"multimodal large language models","score":0.7620000243186951},{"id":"https://openalex.org/keywords/image-to-text-generation","display_name":"image-to-text generation","score":0.6930000185966492},{"id":"https://openalex.org/keywords/text-to-image-generation","display_name":"text-to-image generation","score":0.8700000047683716},{"id":"https://openalex.org/keywords/image-to-image-translation","display_name":"image-to-image translation","score":0.9869999885559082},{"id":"https://openalex.org/keywords/pairwise-comparison","display_name":"pairwise comparison","score":0.6639999747276306}],"concepts":[{"id":"https://openalex.org/C2777286243","wikidata":"https://www.wikidata.org/wiki/Q5591926","display_name":"Grading (engineering)","level":2,"score":0.7154034376144409},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7054686546325684},{"id":"https://openalex.org/C2777146004","wikidata":"https://www.wikidata.org/wiki/Q14949826","display_name":"CLARITY","level":2,"score":0.6331163644790649},{"id":"https://openalex.org/C184898388","wikidata":"https://www.wikidata.org/wiki/Q1435712","display_name":"Pairwise comparison","level":2,"score":0.6036968231201172},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5121212005615234},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.45043396949768066},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.4463924169540405},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.44136306643486023},{"id":"https://openalex.org/C71139939","wikidata":"https://www.wikidata.org/wiki/Q910194","display_name":"Modal","level":2,"score":0.4281437397003174},{"id":"https://openalex.org/C55493867","wikidata":"https://www.wikidata.org/wiki/Q7094","display_name":"Biochemistry","level":1,"score":0.0},{"id":"https://openalex.org/C188027245","wikidata":"https://www.wikidata.org/wiki/Q750446","display_name":"Polymer chemistry","level":1,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.0},{"id":"https://openalex.org/C147176958","wikidata":"https://www.wikidata.org/wiki/Q77590","display_name":"Civil engineering","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2311.01361","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2311.01361","pdf_url":"https://arxiv.org/pdf/2311.01361","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2311.01361","doi":"https://doi.org/10.48550/arxiv.2311.01361","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2311.01361","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2311.01361","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2311.01361","pdf_url":"https://arxiv.org/pdf/2311.01361","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4388327564.pdf","grobid_xml":"https://content.openalex.org/works/W4388327564.grobid-xml"},"referenced_works_count":31,"referenced_works":["https://openalex.org/W1861492603","https://openalex.org/W1956340063","https://openalex.org/W2123301721","https://openalex.org/W2154652894","https://openalex.org/W3094502228","https://openalex.org/W3126337491","https://openalex.org/W3130033937","https://openalex.org/W3135367836","https://openalex.org/W3153469116","https://openalex.org/W3212616005","https://openalex.org/W4224035735","https://openalex.org/W4226278401","https://openalex.org/W4318718936","https://openalex.org/W4322718191","https://openalex.org/W4366850747","https://openalex.org/W4367367040","https://openalex.org/W4380994608","https://openalex.org/W4381253588","https://openalex.org/W4384648625","https://openalex.org/W4384773367","https://openalex.org/W4386076215","https://openalex.org/W4387294588","https://openalex.org/W4387355666","https://openalex.org/W4387929270","https://openalex.org/W4389519254","https://openalex.org/W4390874566","https://openalex.org/W7133185515","https://openalex.org/W7133196460","https://openalex.org/W7133208264","https://openalex.org/W7133231369","https://openalex.org/W7133239970"],"related_works":["https://openalex.org/W4387838608","https://openalex.org/W4387963941","https://openalex.org/W4392576025","https://openalex.org/W4391766363","https://openalex.org/W4406983024","https://openalex.org/W4387294588","https://openalex.org/W4402193259","https://openalex.org/W7160118339","https://openalex.org/W4404450595","https://openalex.org/W4402670365"],"abstract_inverted_index":{"Automatically":[0],"evaluating":[1],"vision-language":[2],"tasks":[3,40,56,94],"is":[4,127],"challenging,":[5],"especially":[6],"when":[7],"it":[8],"comes":[9],"to":[10,15,64,70,119],"reflecting":[11],"human":[12],"judgments":[13],"due":[14],"limitations":[16,107],"in":[17,28],"accounting":[18],"for":[19,38,52,101,129],"fine-grained":[20],"details.":[21],"Although":[22],"GPT-4V":[23,33,86],"has":[24,41],"shown":[25],"promising":[26,88,128],"results":[27],"various":[29,93],"multi-modal":[30,102],"tasks,":[31],"leveraging":[32],"as":[34,104],"a":[35],"generalist":[36],"evaluator":[37],"these":[39],"not":[42],"yet":[43],"been":[44],"systematically":[45],"explored.":[46],"We":[47,73],"comprehensively":[48],"validate":[49],"GPT-4V's":[50],"capabilities":[51],"evaluation":[53,76,96],"purposes,":[54],"addressing":[55],"ranging":[57],"from":[58],"foundational":[59],"image-to-text":[60],"and":[61,68,80,95,113],"text-to-image":[62],"synthesis":[63],"high-level":[65],"image-to-image":[66],"translations":[67],"multi-images":[69],"text":[71],"alignment.":[72],"employ":[74],"two":[75],"methods,":[77,97],"single-answer":[78],"grading":[79,112],"pairwise":[81],"comparison,":[82],"using":[83],"GPT-4V.":[84],"Notably,":[85],"shows":[87],"agreement":[89],"with":[90,124],"humans":[91],"across":[92],"demonstrating":[98],"immense":[99],"potential":[100],"LLMs":[103],"evaluators.":[105],"Despite":[106],"like":[108],"restricted":[109],"visual":[110],"clarity":[111],"real-world":[114],"complex":[115],"reasoning,":[116],"its":[117],"ability":[118],"provide":[120],"human-aligned":[121],"scores":[122],"enriched":[123],"detailed":[125],"explanations":[126],"universal":[130],"automatic":[131],"evaluator.":[132]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W4388327564","counts_by_year":[{"year":2026,"cited_by_count":49},{"year":2025,"cited_by_count":64},{"year":2024,"cited_by_count":51},{"year":2023,"cited_by_count":4}],"updated_date":"2026-11-16T12:05:36.265796","created_date":"2023-11-04T00:00:00"}
