{"id":"https://openalex.org/W4414968467","doi":"https://doi.org/10.1109/iccv51701.2025.00808","title":"Bias in Gender Bias Benchmarks: How Spurious Features Distort Evaluation","display_name":"Bias in Gender Bias Benchmarks: How Spurious Features Distort Evaluation","publication_year":2025,"publication_date":"2025-10-19","ids":{"openalex":"https://openalex.org/W4414968467","doi":"https://doi.org/10.1109/iccv51701.2025.00808"},"language":"en","primary_location":{"id":"doi:10.1109/iccv51701.2025.00808","is_oa":false,"landing_page_url":"https://doi.org/10.1109/iccv51701.2025.00808","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 IEEE/CVF International Conference on Computer Vision (ICCV)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2509.07596","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5068508238","display_name":"Yusuke Hirota","orcid":"https://orcid.org/0000-0002-4661-9584"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yusuke Hirota","raw_affiliation_strings":["NVIDIA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"NVIDIA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5020411666","display_name":"Ryo Hachiuma","orcid":"https://orcid.org/0000-0001-8274-3710"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ryo Hachiuma","raw_affiliation_strings":["NVIDIA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"NVIDIA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101884383","display_name":"Boyi Li","orcid":"https://orcid.org/0000-0002-8921-3808"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Boyi Li","raw_affiliation_strings":["NVIDIA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"NVIDIA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102930940","display_name":"Ximing Lu","orcid":"https://orcid.org/0000-0001-6671-4573"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ximing Lu","raw_affiliation_strings":["NVIDIA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"NVIDIA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5119909762","display_name":"Michael Ross Boone","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Michael Ross Boone","raw_affiliation_strings":["NVIDIA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"NVIDIA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5091869385","display_name":"Boris Ivanovic","orcid":"https://orcid.org/0000-0002-8698-202X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Boris Ivanovic","raw_affiliation_strings":["NVIDIA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"NVIDIA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102992157","display_name":"Yejin Choi","orcid":"https://orcid.org/0000-0003-3032-5378"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yejin Choi","raw_affiliation_strings":["NVIDIA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"NVIDIA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5050003000","display_name":"Marco Pavone","orcid":"https://orcid.org/0000-0002-0206-4337"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Marco Pavone","raw_affiliation_strings":["NVIDIA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"NVIDIA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5090045508","display_name":"Yu-Chiang Frank Wang","orcid":"https://orcid.org/0000-0002-2333-157X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yu-Chiang Frank Wang","raw_affiliation_strings":["NVIDIA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"NVIDIA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5028370193","display_name":"Noa Garc\u00eda","orcid":"https://orcid.org/0000-0002-9200-6359"},"institutions":[{"id":"https://openalex.org/I98285908","display_name":"The University of Osaka","ror":"https://ror.org/035t8zc32","country_code":"JP","type":"education","lineage":["https://openalex.org/I98285908"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Noa Garcia","raw_affiliation_strings":["Osaka University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Osaka University","institution_ids":["https://openalex.org/I98285908"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5110498147","display_name":"Yuta Nakashima","orcid":null},"institutions":[{"id":"https://openalex.org/I98285908","display_name":"The University of Osaka","ror":"https://ror.org/035t8zc32","country_code":"JP","type":"education","lineage":["https://openalex.org/I98285908"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Yuta Nakashima","raw_affiliation_strings":["Osaka University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Osaka University","institution_ids":["https://openalex.org/I98285908"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5020376803","display_name":"Chao-Han Huck Yang","orcid":"https://orcid.org/0000-0003-2879-8811"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chao-Han Huck Yang","raw_affiliation_strings":["NVIDIA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"NVIDIA","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"8634","last_page":"8644"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11432","display_name":"Gender Politics and Representation","score":0.2085999995470047,"subfield":{"id":"https://openalex.org/subfields/3318","display_name":"Gender Studies"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T11432","display_name":"Gender Politics and Representation","score":0.2085999995470047,"subfield":{"id":"https://openalex.org/subfields/3318","display_name":"Gender Studies"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/spurious-relationship","display_name":"Spurious relationship","score":0.9592000246047974},{"id":"https://openalex.org/keywords/gender-bias","display_name":"Gender bias","score":0.7843000292778015},{"id":"https://openalex.org/keywords/masking","display_name":"Masking (illustration)","score":0.6254000067710876},{"id":"https://openalex.org/keywords/response-bias","display_name":"Response bias","score":0.4733000099658966},{"id":"https://openalex.org/keywords/information-bias","display_name":"Information bias","score":0.39969998598098755},{"id":"https://openalex.org/keywords/selection-bias","display_name":"Selection bias","score":0.35030001401901245},{"id":"https://openalex.org/keywords/generative-grammar","display_name":"Generative grammar","score":0.3231000006198883}],"concepts":[{"id":"https://openalex.org/C97256817","wikidata":"https://www.wikidata.org/wiki/Q1462316","display_name":"Spurious relationship","level":2,"score":0.9592000246047974},{"id":"https://openalex.org/C2983427547","wikidata":"https://www.wikidata.org/wiki/Q93200","display_name":"Gender bias","level":2,"score":0.7843000292778015},{"id":"https://openalex.org/C2777402240","wikidata":"https://www.wikidata.org/wiki/Q6783436","display_name":"Masking (illustration)","level":2,"score":0.6254000067710876},{"id":"https://openalex.org/C159447121","wikidata":"https://www.wikidata.org/wiki/Q490535","display_name":"Response bias","level":2,"score":0.4733000099658966},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4377000033855438},{"id":"https://openalex.org/C88629717","wikidata":"https://www.wikidata.org/wiki/Q17056323","display_name":"Information bias","level":3,"score":0.39969998598098755},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.38580000400543213},{"id":"https://openalex.org/C40423286","wikidata":"https://www.wikidata.org/wiki/Q284172","display_name":"Selection bias","level":2,"score":0.35030001401901245},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.34439998865127563},{"id":"https://openalex.org/C39890363","wikidata":"https://www.wikidata.org/wiki/Q36108","display_name":"Generative grammar","level":2,"score":0.3231000006198883},{"id":"https://openalex.org/C75917345","wikidata":"https://www.wikidata.org/wiki/Q2725298","display_name":"Sampling bias","level":3,"score":0.30799999833106995},{"id":"https://openalex.org/C189216375","wikidata":"https://www.wikidata.org/wiki/Q1127759","display_name":"Cognitive bias","level":3,"score":0.302700012922287},{"id":"https://openalex.org/C79585631","wikidata":"https://www.wikidata.org/wiki/Q431498","display_name":"Confirmation bias","level":2,"score":0.28839999437332153},{"id":"https://openalex.org/C2780439572","wikidata":"https://www.wikidata.org/wiki/Q919364","display_name":"Publication bias","level":3,"score":0.28029999136924744},{"id":"https://openalex.org/C2991991027","wikidata":"https://www.wikidata.org/wiki/Q6007314","display_name":"Implicit bias","level":2,"score":0.27219998836517334},{"id":"https://openalex.org/C2776650193","wikidata":"https://www.wikidata.org/wiki/Q264661","display_name":"Obstacle","level":2,"score":0.2671000063419342}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1109/iccv51701.2025.00808","is_oa":false,"landing_page_url":"https://doi.org/10.1109/iccv51701.2025.00808","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 IEEE/CVF International Conference on Computer Vision (ICCV)","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2509.07596","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2509.07596","pdf_url":"https://arxiv.org/pdf/2509.07596","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2509.07596","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2509.07596","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2509.07596","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2509.07596","pdf_url":"https://arxiv.org/pdf/2509.07596","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Gender":[0],"bias":[1,50,57,86,109,129,156,166],"in":[2,48,117,122],"vision-language":[3],"foundation":[4],"models":[5],"(VLMs)":[6],"raises":[7],"concerns":[8],"about":[9],"their":[10,83,143],"safe":[11],"deployment":[12],"and":[13,35,41,76,78,120],"is":[14,150],"typically":[15],"evaluated":[16],"using":[17],"benchmarks":[18,28,72,149],"with":[19],"gender":[20,34,49,56,140],"annotations":[21],"on":[22,85],"real-world":[23],"images.":[24],"However,":[25],"as":[26,39,96],"these":[27],"often":[29,131],"contain":[30],"spurious":[31,53,136,147],"correlations":[32],"between":[33],"non-gender":[36,66],"features,":[37],"such":[38,95],"objects":[40,101],"backgrounds,":[42,105],"we":[43,63,153],"identify":[44],"a":[45,163],"critical":[46],"oversight":[47],"evaluation:":[51],"Do":[52],"features":[54,67,137],"distort":[55],"evaluation?":[58],"To":[59],"address":[60],"this":[61],"question,":[62],"systematically":[64],"perturb":[65],"across":[68],"four":[69],"widely":[70],"used":[71],"(COCO-gender,":[73],"FACET,":[74],"MIAP,":[75],"PHASE)":[77],"various":[79],"VLMs":[80,119],"to":[81,115,135,161],"quantify":[82],"impact":[84],"evaluation.":[87],"Our":[88],"findings":[89],"reveal":[90],"that":[91,127],"even":[92],"minimal":[93],"perturbations,":[94],"masking":[97],"just":[98],"10%":[99],"of":[100],"or":[102],"weakly":[103],"blurring":[104],"can":[106],"dramatically":[107],"alter":[108],"scores,":[110],"shifting":[111],"metrics":[112,157],"by":[113],"up":[114],"175%":[116],"generative":[118],"43%":[121],"CLIP":[123],"variants.":[124],"This":[125],"suggests":[126],"current":[128],"evaluations":[130],"reflect":[132],"model":[133],"responses":[134],"rather":[138],"than":[139],"bias,":[141],"undermining":[142],"reliability.":[144],"Since":[145],"creating":[146],"feature-free":[148],"fundamentally":[151],"challenging,":[152],"recommend":[154],"reporting":[155],"alongside":[158],"feature-sensitivity":[159],"measurements":[160],"enable":[162],"more":[164],"reliable":[165],"assessment.":[167]},"counts_by_year":[],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-09T00:00:00"}