{"id":"https://openalex.org/W4412888105","doi":"https://doi.org/10.18653/v1/2025.findings-acl.955","title":"Biases Propagate in Encoder-based Vision-Language Models: A Systematic Analysis From Intrinsic Measures to Zero-shot Retrieval Outcomes","display_name":"Biases Propagate in Encoder-based Vision-Language Models: A Systematic Analysis From Intrinsic Measures to Zero-shot Retrieval Outcomes","publication_year":2025,"publication_date":"2025-01-01","ids":{"openalex":"https://openalex.org/W4412888105","doi":"https://doi.org/10.18653/v1/2025.findings-acl.955"},"language":"en","primary_location":{"id":"doi:10.18653/v1/2025.findings-acl.955","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.findings-acl.955","pdf_url":"https://aclanthology.org/2025.findings-acl.955.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: ACL 2025","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://aclanthology.org/2025.findings-acl.955.pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5087599726","display_name":"Kshitish Ghate","orcid":null},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kshitish Ghate","raw_affiliation_strings":["Carnegie Mellon University Northwestern University Carnegie Mellon University University of Washington"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University Northwestern University Carnegie Mellon University University of Washington","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5004306822","display_name":"Tessa Elizabeth Sadie Charlesworth","orcid":"https://orcid.org/0000-0001-5048-3088"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Tessa Charlesworth","raw_affiliation_strings":["Carnegie Mellon University Northwestern University Carnegie Mellon University University of Washington"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University Northwestern University Carnegie Mellon University University of Washington","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5038581447","display_name":"Mona Diab","orcid":"https://orcid.org/0000-0002-7696-1436"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Mona T. Diab","raw_affiliation_strings":["Carnegie Mellon University Northwestern University Carnegie Mellon University University of Washington"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University Northwestern University Carnegie Mellon University University of Washington","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5101545719","display_name":"Aylin Caliskan","orcid":"https://orcid.org/0000-0001-7154-8629"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Aylin Caliskan","raw_affiliation_strings":["Carnegie Mellon University Northwestern University Carnegie Mellon University University of Washington"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University Northwestern University Carnegie Mellon University University of Washington","institution_ids":["https://openalex.org/I74973139"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I74973139"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":2,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"18562","last_page":"18580"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9835000038146973,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9835000038146973,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.9222999811172485,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/zero","display_name":"Zero (linguistics)","score":0.6970415115356445},{"id":"https://openalex.org/keywords/encoder","display_name":"Encoder","score":0.6826720833778381},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6622064113616943},{"id":"https://openalex.org/keywords/shot","display_name":"Shot (pellet)","score":0.5682436227798462},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4808114767074585},{"id":"https://openalex.org/keywords/computer-vision","display_name":"Computer vision","score":0.36330291628837585},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.3361976146697998},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.326969712972641},{"id":"https://openalex.org/keywords/linguistics","display_name":"Linguistics","score":0.09083402156829834}],"concepts":[{"id":"https://openalex.org/C2780813799","wikidata":"https://www.wikidata.org/wiki/Q3274237","display_name":"Zero (linguistics)","level":2,"score":0.6970415115356445},{"id":"https://openalex.org/C118505674","wikidata":"https://www.wikidata.org/wiki/Q42586063","display_name":"Encoder","level":2,"score":0.6826720833778381},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6622064113616943},{"id":"https://openalex.org/C2778344882","wikidata":"https://www.wikidata.org/wiki/Q278938","display_name":"Shot (pellet)","level":2,"score":0.5682436227798462},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4808114767074585},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.36330291628837585},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3361976146697998},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.326969712972641},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.09083402156829834},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C178790620","wikidata":"https://www.wikidata.org/wiki/Q11351","display_name":"Organic chemistry","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.18653/v1/2025.findings-acl.955","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.findings-acl.955","pdf_url":"https://aclanthology.org/2025.findings-acl.955.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: ACL 2025","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.18653/v1/2025.findings-acl.955","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.findings-acl.955","pdf_url":"https://aclanthology.org/2025.findings-acl.955.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: ACL 2025","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G8134836207","display_name":"PURPOSE: ARTIFICIAL INTELLIGENCE (AI) IS WIDESPREAD IN TODAY'S SOCIETY, WITH AI ALGORITHMS BEING INCREASINGLY RELIED ON TO MAKE CONSEQUENTIAL DECISIONS ABOUT INDIVIDUALS. ALTHOUGH THERE WAS INITIAL OPTIMISM THAT AI APPLICATIONS WOULD MAKE A MORE EFFICIENT AND FAIR SOCIETY, ANECDOTAL EXAMPLES OF BIASED OUTCOMES IN AI APPLICATIONS NOW ABOUND FROM DOMAINS AS WIDE AS WEB SEARCH, MACHINE TRANSLATION, UNIVERSITY ADMISSIONS, RESUME SCREENING, AND JOB CANDIDATE ASSESSMENT. YET, BEYOND THESE ANECDOTES, THERE REMAINS NO COMPREHENSIVE STUDY THAT EXAMINES (1) HOW THE EXTENT OF BIASES IN GENERAL-PURPOSE MODALITY REPRESENTATIONS THAT AI RELIES ON PROPAGATE TO DOWNSTREAM DECISIONS, (2) HOW BIASED AI DECISIONS INTERACT WITH THE BIASES OF HUMANS-IN-THE-LOOP, AND (3) HOW AI-MEDIATED OUTCOMES IMPACT THE BIASES OF FUTURE GENERATIONS OF AI MODELS. THE PROPOSED RESEARCH SEEKS TO ADDRESS THESE NEEDS, THEREBY CONTRIBUTING TO NIST'S TRUSTWORTHY AND RESPONSIBLE AI EFFORTS. ACTIVITIES TO BE PERFORMED: IN THIS PROJECT, THE PROPOSERS WILL ANALYZE THE IMPACT OF BIASED GENERAL-PURPOSE MACHINE REPRESENTATIONS GENERATED VIA UNSUPERVISED MACHINE LEARNING ON THE AUTOMATED DECISIONS OF SUPERVISED MACHINE LEARNING ALGORITHMS THAT RELY ON STRUCTURED DATA IN THE EMPLOYMENT DOMAIN. AS A STARTING POINT, THE PROPOSERS WILL FOCUS ON GENDER AND RACIAL BIAS IN RESUME SCREENING AND JOB CANDIDATE ASSESSMENT, AN APPLICATION DOMAIN WITH CLEAR CONSEQUENCES FOR LIFE'S OPPORTUNITIES AND OUTCOMES. THE PROPOSERS WILL DEVELOP A BIAS PROPAGATION AND INTERACTION AUDIT FRAMEWORK THAT GENERALIZES ACROSS MODALITIES, BIAS TYPES, AND APPLICATION DOMAINS. EXPECTED OUTCOMES: THE PROPOSED RESEARCH WILL CONDUCT TWO COMPLEMENTARY STUDIES TO ANSWER THE FOLLOWING IMPORTANT QUESTIONS: (1) GIVEN THAT BIAS REFLECTED FROM SOCIETY TO LARGE-SCALE DATASETS GETS EMBEDDED IN GENERAL-PURPOSE MACHINE REPRESENTATIONS USED FOR PROCESSING DATA, HOW DOES REPRESENTATIONAL BIAS PROPAGATE TO DOWNSTREAM APPLICATIONS? (2) HOW ARE HUMANS-IN-THE-LOOP COLLABORATING WITH AI IN DECISION MAKING AFFECTED BY THE BIASED OUTCOMES OF MACHINES,GIVEN THEIR OWN IMPLICIT BIASES? THE RESEARCH STUDIES SUPPORTED BY THIS GRANT WILL BE PUBLISHED IN THE SCIENTIFIC LITERATURE, SPECIFICALLY BY CONTRIBUTING PAPERS ON AN ANNUAL BASIS AT ARTIFICIAL INTELLIGENCE AND FAIRNESS VENUES FOR COMPUTER AND INFORMATION SCIENCES. INTENDED BENEFICIARIES: SOCIOTECHNICAL ARTIFICIAL INTELLIGENCE (AI) SYSTEMS WITH HUMANS-IN-THE-LOOP MAKE CONSEQUENTIAL DECISIONS ABOUT INDIVIDUALS IN SCENARIOS THAT DETERMINE LIFE'S OPPORTUNITIES AND OUTCOMES. DEVELOPING METHODS AND METRICS TO MEASURE HOW THE BIASES OF SOCIETY, AI, AND HUMANS-IN-THE-LOOP INTERACT IN DECISION MAKING WILL PROVIDE QUANTITATIVE EVIDENCE ON THE IMPACT OF AI ON SOCIETY. THIS PROJECT WILL BENEFIT THE GENERAL PUBLIC BY PROVIDING MUCH-NEEDED RESEARCH TO SUPPORT DEVELOPING STANDARDS FOR IDENTIFYING AND MANAGING BIAS IN AI. FAIR WITH HARMFUL BIAS MANAGED IS A KEY CHARACTERISTIC OF TRUSTWORTHY AI AS DEFINED IN NIST'S AI RISK MANAGEMENT FRAMEWORK, THEREFORE THIS BIAS RESEARCH WILL CLEARLY BENEFIT NIST'S LARGER TRUSTWORTHY AND RESPONSIBLE AI PROGRAM AND ADVANCE THE STATE-OF-THE-ART FOR BIAS EVALUATIONS IN AI SYSTEMS. SUBRECIPIENT ACTIVITIES: THERE ARE NO PLANNED SUBAWARDS.","funder_award_id":"60NANB23D194","funder_id":"https://openalex.org/F4320332178","funder_display_name":"National Institute of Standards and Technology"}],"funders":[{"id":"https://openalex.org/F4320332178","display_name":"National Institute of Standards and Technology","ror":"https://ror.org/05xpvk416"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4412888105.pdf","grobid_xml":"https://content.openalex.org/works/W4412888105.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W2074502265","https://openalex.org/W4214877189","https://openalex.org/W2773965352","https://openalex.org/W2381179799","https://openalex.org/W4411535265","https://openalex.org/W2980279061","https://openalex.org/W2334685461","https://openalex.org/W2366718574","https://openalex.org/W2359774528","https://openalex.org/W4298312966"],"abstract_inverted_index":{"To":[0],"build":[1],"fair":[2],"AI":[3,136],"systems":[4],"we":[5,27,116],"need":[6],"to":[7,13,57,143],"understand":[8],"how":[9,45],"social-group":[10],"biases":[11,21,31,48],"intrinsic":[12,30,64,90],"foundational":[14],"encoder-based":[15],"vision-language":[16],"models":[17,120],"(VLMs)":[18],"manifest":[19],"in":[20,22,32,68,78],"downstream":[23],"tasks.In":[24],"this":[25,59],"study,":[26],"demonstrate":[28],"that":[29,118,127,153],"VLM":[33],"representations":[34],"systematically":[35],"\"carry":[36],"over\"":[37],"or":[38],"propagate":[39],"into":[40],"zero-shot":[41,79],"retrieval":[42,107],"tasks,":[43],"revealing":[44],"deeply":[46],"rooted":[47],"shape":[49],"a":[50,54,125],"model's":[51],"outputs.We":[52],"introduce":[53],"controlled":[55],"framework":[56,138],"measure":[58,144],"propagation":[60,146],"by":[61],"correlating":[62],"(a)":[63],"measures":[65,75],"of":[66,76,147],"bias":[67,77,123],"the":[69,131,145],"representational":[70],"space":[71],"with":[72,94],"(b)":[73],"extrinsic":[74,92],"text-to-image":[80],"(TTI)":[81],"and":[82,91,112,149],"image-to-text":[83],"(ITT)":[84],"retrieval.Results":[85],"show":[86],"substantial":[87],"correlations":[88],"between":[89],"bias,":[93],"an":[95],"average":[96],"=":[97],"0.83":[98],"0.10.This":[99],"pattern":[100],"is":[101],"consistent":[102],"across":[103],"114":[104],"analyses,":[105],"both":[106],"directions,":[108],"six":[109],"social":[110],"groups,":[111],"three":[113],"distinct":[114],"VLMs.Notably,":[115],"find":[117],"larger/better-performing":[119],"exhibit":[121],"greater":[122],"propagation,":[124,159],"finding":[126],"raises":[128],"concerns":[129],"given":[130],"trend":[132],"towards":[133],"increasingly":[134],"complex":[135],"models.Our":[137],"introduces":[139],"baseline":[140],"evaluation":[141],"tasks":[142],"group":[148],"valence":[150],"signals.Investigations":[151],"reveal":[152],"underrepresented":[154],"groups":[155],"experience":[156],"less":[157],"robust":[158],"further":[160],"skewing":[161],"their":[162],"model-related":[163],"outcomes.":[164]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":1}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}