{"id":"https://openalex.org/W7166573239","doi":"https://doi.org/10.1145/3805689.3806459","title":"Automated bias blind spots: Examining stereotype associations in VLM-as-a-judge paradigms","display_name":"Automated bias blind spots: Examining stereotype associations in VLM-as-a-judge paradigms","publication_year":2026,"publication_date":"2026-06-25","ids":{"openalex":"https://openalex.org/W7166573239","doi":"https://doi.org/10.1145/3805689.3806459"},"language":null,"primary_location":{"id":"doi:10.1145/3805689.3806459","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3805689.3806459","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2026 ACM Conference on Fairness, Accountability, and Transparency","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3805689.3806459","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5052083701","display_name":"Rachel Hong","orcid":"https://orcid.org/0009-0005-4275-653X"},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Rachel Hong","raw_affiliation_strings":["University of Washington, Seattle, Washington, USA"],"raw_orcid":"https://orcid.org/0009-0005-4275-653X","affiliations":[{"raw_affiliation_string":"University of Washington, Seattle, Washington, USA","institution_ids":["https://openalex.org/I201448701"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5064234845","display_name":"Anaelia Ovalle","orcid":"https://orcid.org/0000-0002-0531-7520"},"institutions":[{"id":"https://openalex.org/I4210112286","display_name":"The Metropolitan Opera (United States)","ror":"https://ror.org/022q3vw94","country_code":"US","type":"company","lineage":["https://openalex.org/I4210112286"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Anaelia Ovalle","raw_affiliation_strings":["FAIR at Meta, New York, New York, USA"],"raw_orcid":"https://orcid.org/0000-0002-0531-7520","affiliations":[{"raw_affiliation_string":"FAIR at Meta, New York, New York, USA","institution_ids":["https://openalex.org/I4210112286"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5067335278","display_name":"Megan Ung","orcid":"https://orcid.org/0000-0003-3063-3207"},"institutions":[{"id":"https://openalex.org/I4210112286","display_name":"The Metropolitan Opera (United States)","ror":"https://ror.org/022q3vw94","country_code":"US","type":"company","lineage":["https://openalex.org/I4210112286"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Megan Ung","raw_affiliation_strings":["FAIR at Meta, New York, New York, USA"],"raw_orcid":"https://orcid.org/0000-0003-3063-3207","affiliations":[{"raw_affiliation_string":"FAIR at Meta, New York, New York, USA","institution_ids":["https://openalex.org/I4210112286"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139623012","display_name":"Evangelia Spiliopoulou","orcid":"https://orcid.org/0009-0009-7269-6813"},"institutions":[{"id":"https://openalex.org/I4210112286","display_name":"The Metropolitan Opera (United States)","ror":"https://ror.org/022q3vw94","country_code":"US","type":"company","lineage":["https://openalex.org/I4210112286"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Evangelia Spiliopoulou","raw_affiliation_strings":["FAIR at Meta, New York, New York, USA"],"raw_orcid":"https://orcid.org/0009-0009-7269-6813","affiliations":[{"raw_affiliation_string":"FAIR at Meta, New York, New York, USA","institution_ids":["https://openalex.org/I4210112286"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5071931582","display_name":"Levent Sagun","orcid":"https://orcid.org/0000-0001-5403-4124"},"institutions":[{"id":"https://openalex.org/I4210112279","display_name":"Milieux environnementaux, transferts et interactions dans les hydrosyst\u00e8mes et les sols","ror":"https://ror.org/0229z0679","country_code":"FR","type":"facility","lineage":["https://openalex.org/I1294671590","https://openalex.org/I1294671590","https://openalex.org/I159885104","https://openalex.org/I2746051580","https://openalex.org/I39804081","https://openalex.org/I4210107625","https://openalex.org/I4210112279"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Levent Sagun","raw_affiliation_strings":["FAIR at Meta, Paris, France"],"raw_orcid":"https://orcid.org/0000-0001-5403-4124","affiliations":[{"raw_affiliation_string":"FAIR at Meta, Paris, France","institution_ids":["https://openalex.org/I4210112279"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5062696000","display_name":"Adina Williams","orcid":"https://orcid.org/0000-0001-5281-3343"},"institutions":[{"id":"https://openalex.org/I4210112286","display_name":"The Metropolitan Opera (United States)","ror":"https://ror.org/022q3vw94","country_code":"US","type":"company","lineage":["https://openalex.org/I4210112286"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Adina Williams","raw_affiliation_strings":["FAIR at Meta, New York, New York, USA"],"raw_orcid":"https://orcid.org/0000-0001-5281-3343","affiliations":[{"raw_affiliation_string":"FAIR at Meta, New York, New York, USA","institution_ids":["https://openalex.org/I4210112286"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5139618328","display_name":"Candace Ross","orcid":"https://orcid.org/0000-0002-6715-8086"},"institutions":[{"id":"https://openalex.org/I4210112286","display_name":"The Metropolitan Opera (United States)","ror":"https://ror.org/022q3vw94","country_code":"US","type":"company","lineage":["https://openalex.org/I4210112286"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Candace Ross","raw_affiliation_strings":["FAIR at Meta, New York, New York, USA"],"raw_orcid":"https://orcid.org/0000-0002-6715-8086","affiliations":[{"raw_affiliation_string":"FAIR at Meta, New York, New York, USA","institution_ids":["https://openalex.org/I4210112286"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"6510","last_page":"6553"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10314","display_name":"Social and Intergroup Psychology","score":0.13300000131130219,"subfield":{"id":"https://openalex.org/subfields/3312","display_name":"Sociology and Political Science"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10314","display_name":"Social and Intergroup Psychology","score":0.13300000131130219,"subfield":{"id":"https://openalex.org/subfields/3312","display_name":"Sociology and Political Science"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10525","display_name":"Human-Automation Interaction and Safety","score":0.06790000200271606,"subfield":{"id":"https://openalex.org/subfields/3207","display_name":"Social Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11148","display_name":"Language, Metaphor, and Cognition","score":0.03709999844431877,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/stereotype","display_name":"Stereotype (UML)","score":0.3693999946117401},{"id":"https://openalex.org/keywords/perception","display_name":"Perception","score":0.3395000100135803},{"id":"https://openalex.org/keywords/affect","display_name":"Affect (linguistics)","score":0.29490000009536743},{"id":"https://openalex.org/keywords/identification","display_name":"Identification (biology)","score":0.2750999927520752},{"id":"https://openalex.org/keywords/perspective","display_name":"Perspective (graphical)","score":0.2655999958515167}],"concepts":[{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.6054999828338623},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.5041000247001648},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.38909998536109924},{"id":"https://openalex.org/C168127410","wikidata":"https://www.wikidata.org/wiki/Q1754331","display_name":"Stereotype (UML)","level":2,"score":0.3693999946117401},{"id":"https://openalex.org/C26760741","wikidata":"https://www.wikidata.org/wiki/Q160402","display_name":"Perception","level":2,"score":0.3395000100135803},{"id":"https://openalex.org/C2776035688","wikidata":"https://www.wikidata.org/wiki/Q1606558","display_name":"Affect (linguistics)","level":2,"score":0.29490000009536743},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.2903999984264374},{"id":"https://openalex.org/C116834253","wikidata":"https://www.wikidata.org/wiki/Q2039217","display_name":"Identification (biology)","level":2,"score":0.2750999927520752},{"id":"https://openalex.org/C12713177","wikidata":"https://www.wikidata.org/wiki/Q1900281","display_name":"Perspective (graphical)","level":2,"score":0.2655999958515167},{"id":"https://openalex.org/C138496976","wikidata":"https://www.wikidata.org/wiki/Q175002","display_name":"Developmental psychology","level":1,"score":0.2653999924659729},{"id":"https://openalex.org/C165064840","wikidata":"https://www.wikidata.org/wiki/Q1321061","display_name":"Matching (statistics)","level":2,"score":0.2540999948978424},{"id":"https://openalex.org/C2983427547","wikidata":"https://www.wikidata.org/wiki/Q93200","display_name":"Gender bias","level":2,"score":0.25270000100135803}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3805689.3806459","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3805689.3806459","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2026 ACM Conference on Fairness, Accountability, and Transparency","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3805689.3806459","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3805689.3806459","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2026 ACM Conference on Fairness, Accountability, and Transparency","raw_type":"proceedings-article"},"sustainable_development_goals":[{"display_name":"Reduced inequalities","score":0.5205754041671753,"id":"https://metadata.un.org/sdg/10"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":55,"referenced_works":["https://openalex.org/W2036469702","https://openalex.org/W3103639864","https://openalex.org/W3172141633","https://openalex.org/W3172415559","https://openalex.org/W3207316473","https://openalex.org/W4205570006","https://openalex.org/W4285294416","https://openalex.org/W4287887133","https://openalex.org/W4385571220","https://openalex.org/W4385571232","https://openalex.org/W4385571830","https://openalex.org/W4385573981","https://openalex.org/W4385574250","https://openalex.org/W4386065884","https://openalex.org/W4386249234","https://openalex.org/W4387321091","https://openalex.org/W4389518686","https://openalex.org/W4389524209","https://openalex.org/W4390873440","https://openalex.org/W4393160418","https://openalex.org/W4399356353","https://openalex.org/W4401043020","https://openalex.org/W4402670679","https://openalex.org/W4404780679","https://openalex.org/W4404782878","https://openalex.org/W4407662298","https://openalex.org/W4409748981","https://openalex.org/W4411113083","https://openalex.org/W4411119351","https://openalex.org/W4411120637","https://openalex.org/W4411541907","https://openalex.org/W4411542000","https://openalex.org/W4411549853","https://openalex.org/W4411550447","https://openalex.org/W4411550524","https://openalex.org/W4411630213","https://openalex.org/W4411630225","https://openalex.org/W4411638697","https://openalex.org/W4412886673","https://openalex.org/W4412886926","https://openalex.org/W4412888105","https://openalex.org/W4412888679","https://openalex.org/W4412944664","https://openalex.org/W4412944835","https://openalex.org/W4413325827","https://openalex.org/W4415797148","https://openalex.org/W4416034226","https://openalex.org/W4416037051","https://openalex.org/W4416037109","https://openalex.org/W7133228402","https://openalex.org/W7133230326","https://openalex.org/W7133236450","https://openalex.org/W7133239970","https://openalex.org/W7138303952","https://openalex.org/W7160207210"],"related_works":[],"abstract_inverted_index":{"Large":[0],"language":[1],"models":[2,6,130,226,260,267,293,302],"(LLMs)":[3],"and":[4,18,23,35,178,190,217,315,335],"vision-language":[5],"(VLMs)":[7],"are":[8,27,76,227,254],"increasingly":[9],"being":[10],"employed":[11],"to":[12,87,113,137,157,207,214,231,266,278,311,327,330],"judge":[13,131,219,236],"model":[14,126,134,154],"responses":[15,34,135,155],"through":[16,149],"LLM-as-a-judge":[17],"VLM-as-a-judge":[19],"paradigms.":[20],"The":[21],"validity":[22],"effectiveness":[24],"of":[25,125,163,197,204,282,292,313,324],"judges":[26,152,253,287,325],"often":[28],"based":[29,48,306],"on":[30,37,49,294,307],"agreement":[31,341],"with":[32,342],"human":[33,343],"performance":[36,291],"verifiable":[38],"benchmarks.":[39,296],"However,":[40],"it":[41],"is":[42,95,202],"unclear":[43],"whether":[44],"a":[45,55,59,70,96,140],"model's":[46],"behavior":[47,127],"its":[50],"role":[51,237],"-":[52,61,175,238],"when":[53,81,129],"completing":[54],"task":[56,60],"versus":[57],"judging":[58,82,268],"has":[62],"been":[63],"explored,":[64],"particularly":[65,128],"in":[66,72,91,142,169,224,234,259,345],"socially-grounded":[67],"contexts.":[68],"As":[69],"parallel":[71],"cognitive":[73,97,210],"science,":[74],"humans":[75],"adept":[77],"at":[78,273],"recognizing":[79],"biases":[80,90,233,281],"other":[83,346],"humans,":[84],"yet":[85],"fail":[86],"recognize":[88],"similar":[89],"themselves.":[92],"This":[93],"asymmetry":[94,124],"phenomenon":[98],"known":[99],"as":[100],"the":[101,118,123,164,195,205,209,235,245,279,290,308],"bias":[102,120,166,211,241,295],"blind":[103,121,167,212,242],"spot.":[104],"Motivated":[105],"by":[106,186],"this,":[107],"we":[108,116,284],"propose":[109],"an":[110],"empirical":[111],"framework":[112,145,181],"test":[114],"what":[115],"deem":[117],"automated":[119,165,240],"spot,":[122],"potentially":[132],"stereotype-reinforcing":[133],"compared":[136],"directly":[138],"generating":[139,184],"response;":[141],"particular,":[143],"our":[144,198,200,318],"ensures":[146],"valid":[147],"comparisons":[148],"probing":[150],"how":[151],"rate":[153],"corresponding":[156],"every":[158],"option.":[159],"We":[160,221],"explore":[161],"potential":[162],"spot":[168,213],"VLMs":[170],"across":[171],"three":[172],"different":[173,183],"modalities":[174],"language-only,":[176],"vision-only,":[177],"language-vision.":[179],"Our":[180],"encompasses":[182],"paradigms,":[185],"incorporating":[187],"chain-of-thought":[188],"reasoning":[189],"verbalized":[191],"confidence":[192],"scores.":[193],"To":[194],"best":[196],"knowledge,":[199],"work":[201],"one":[203],"first":[206],"connect":[208],"assess":[215],"LLM":[216],"VLM":[218],"reliability.":[220],"observe":[222],"that":[223,252,286,301],"aggregate,":[225],"generally":[228],"more":[229],"robust":[230],"social":[232],"demonstrating":[239],"spots.":[243],"At":[244],"same":[246,270],"time,":[247],"this":[248],"does":[249],"not":[250],"imply":[251],"completely":[255],"unbiased;":[256],"stereotypes":[257],"manifested":[258],"answering":[261],"particular":[262],"questions":[263,312],"still":[264],"propagate":[265],"those":[269],"questions,":[271],"although":[272],"lower":[274],"degrees.":[275],"In":[276],"addition":[277],"inconsistent":[280],"judges,":[283],"find":[285],"substantially":[288],"overestimate":[289],"These":[297],"results":[298],"therefore":[299],"indicate":[300],"exhibit":[303],"behaviors":[304],"inconsistently":[305],"role,":[309],"leading":[310],"reliability":[314],"validity.":[316],"Altogether":[317],"findings":[319],"call":[320],"for":[321],"comprehensive":[322],"evaluation":[323],"prior":[326],"using":[328],"them":[329],"measure":[331],"socially":[332],"grounded":[333],"concepts":[334],"encourage":[336],"evaluations":[337],"beyond":[338],"just":[339],"high":[340],"evaluators":[344],"domains.":[347]},"counts_by_year":[],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2026-06-30T00:00:00"}