{"id":"https://openalex.org/W4411550451","doi":"https://doi.org/10.1145/3715275.3732147","title":"Randomness, Not Representation: The Unreliability of Evaluating Cultural Alignment in LLMs","display_name":"Randomness, Not Representation: The Unreliability of Evaluating Cultural Alignment in LLMs","publication_year":2025,"publication_date":"2025-06-23","ids":{"openalex":"https://openalex.org/W4411550451","doi":"https://doi.org/10.1145/3715275.3732147"},"language":"en","primary_location":{"id":"doi:10.1145/3715275.3732147","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3715275.3732147","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3715275.3732147","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2025 ACM Conference on Fairness, Accountability, and Transparency","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3715275.3732147","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5053578798","display_name":"Ariba Khan","orcid":"https://orcid.org/0000-0002-7613-5636"},"institutions":[{"id":"https://openalex.org/I4210110987","display_name":"IIT@MIT","ror":"https://ror.org/01wp8zh54","country_code":"US","type":"facility","lineage":["https://openalex.org/I30771326","https://openalex.org/I4210110987"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Ariba Khan","raw_affiliation_strings":["MIT, Cambridge, USA"],"raw_orcid":"https://orcid.org/0000-0002-7613-5636","affiliations":[{"raw_affiliation_string":"MIT, Cambridge, USA","institution_ids":["https://openalex.org/I4210110987"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5099024522","display_name":"Stephen Casper","orcid":"https://orcid.org/0000-0003-0084-1937"},"institutions":[{"id":"https://openalex.org/I4210110987","display_name":"IIT@MIT","ror":"https://ror.org/01wp8zh54","country_code":"US","type":"facility","lineage":["https://openalex.org/I30771326","https://openalex.org/I4210110987"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Stephen Casper","raw_affiliation_strings":["MIT, Cambridge, USA"],"raw_orcid":"https://orcid.org/0000-0003-0084-1937","affiliations":[{"raw_affiliation_string":"MIT, Cambridge, USA","institution_ids":["https://openalex.org/I4210110987"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5076757561","display_name":"Dylan Hadfield-Menell","orcid":"https://orcid.org/0000-0002-6168-4763"},"institutions":[{"id":"https://openalex.org/I4210110987","display_name":"IIT@MIT","ror":"https://ror.org/01wp8zh54","country_code":"US","type":"facility","lineage":["https://openalex.org/I30771326","https://openalex.org/I4210110987"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Dylan Hadfield-Menell","raw_affiliation_strings":["MIT, Cambridge, USA"],"raw_orcid":"https://orcid.org/0000-0002-6168-4763","affiliations":[{"raw_affiliation_string":"MIT, Cambridge, USA","institution_ids":["https://openalex.org/I4210110987"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5053578798"],"corresponding_institution_ids":["https://openalex.org/I4210110987"],"apc_list":null,"apc_paid":null,"fwci":53.1842,"has_fulltext":true,"cited_by_count":7,"citation_normalized_percentile":{"value":0.99808058,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"2151","last_page":"2165"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13643","display_name":"Artificial Intelligence in Law","score":0.9527999758720398,"subfield":{"id":"https://openalex.org/subfields/3320","display_name":"Political Science and International Relations"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T13643","display_name":"Artificial Intelligence in Law","score":0.9527999758720398,"subfield":{"id":"https://openalex.org/subfields/3320","display_name":"Political Science and International Relations"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12755","display_name":"Legal Education and Practice Innovations","score":0.9488999843597412,"subfield":{"id":"https://openalex.org/subfields/3308","display_name":"Law"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T13851","display_name":"Law, AI, and Intellectual Property","score":0.9272000193595886,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/randomness","display_name":"Randomness","score":0.7944927215576172},{"id":"https://openalex.org/keywords/representation","display_name":"Representation (politics)","score":0.6231507062911987},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.49971652030944824},{"id":"https://openalex.org/keywords/political-science","display_name":"Political science","score":0.21909281611442566},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.12000837922096252},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.10324794054031372}],"concepts":[{"id":"https://openalex.org/C125112378","wikidata":"https://www.wikidata.org/wiki/Q176640","display_name":"Randomness","level":2,"score":0.7944927215576172},{"id":"https://openalex.org/C2776359362","wikidata":"https://www.wikidata.org/wiki/Q2145286","display_name":"Representation (politics)","level":3,"score":0.6231507062911987},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.49971652030944824},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.21909281611442566},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.12000837922096252},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.10324794054031372},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C94625758","wikidata":"https://www.wikidata.org/wiki/Q7163","display_name":"Politics","level":2,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3715275.3732147","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3715275.3732147","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3715275.3732147","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2025 ACM Conference on Fairness, Accountability, and Transparency","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3715275.3732147","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3715275.3732147","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3715275.3732147","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2025 ACM Conference on Fairness, Accountability, and Transparency","raw_type":"proceedings-article"},"sustainable_development_goals":[{"score":0.6800000071525574,"id":"https://metadata.un.org/sdg/10","display_name":"Reduced inequalities"}],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4411550451.pdf","grobid_xml":"https://content.openalex.org/works/W4411550451.grobid-xml"},"referenced_works_count":21,"referenced_works":["https://openalex.org/W1583372272","https://openalex.org/W2021358508","https://openalex.org/W2123791568","https://openalex.org/W2144416034","https://openalex.org/W2895010120","https://openalex.org/W3040416015","https://openalex.org/W3110946289","https://openalex.org/W3113449840","https://openalex.org/W4294415280","https://openalex.org/W4386566829","https://openalex.org/W4386567020","https://openalex.org/W4402565126","https://openalex.org/W4402671261","https://openalex.org/W4402671433","https://openalex.org/W4402671575","https://openalex.org/W4402683988","https://openalex.org/W4404132421","https://openalex.org/W4404782660","https://openalex.org/W4404782944","https://openalex.org/W4410357609","https://openalex.org/W6923535430"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W3034924094","https://openalex.org/W3094954546","https://openalex.org/W1488708774","https://openalex.org/W1982811510","https://openalex.org/W4391100477","https://openalex.org/W2402189625","https://openalex.org/W4327779705"],"abstract_inverted_index":{"Research":[0],"on":[1,76,87,162],"the":[2,142,198],"'cultural":[3],"alignment'":[4],"of":[5,60,66,80,110,118,144,172,182,192,201],"Large":[6],"Language":[7],"Models":[8],"(LLMs)":[9],"has":[10],"emerged":[11],"in":[12,17,154,158],"response":[13],"to":[14,24,98,147,151,177,196],"growing":[15],"interest":[16],"understanding":[18],"representation":[19],"across":[20,120],"diverse":[21],"stakeholders.Current":[22],"approaches":[23,195],"evaluating":[25,197],"cultural":[26,55,101,128,184,199],"alignment":[27,56,72,83,185,200],"through":[28],"survey-based":[29,49,194],"assessments":[30],"that":[31,54,71,85,92,137,165],"borrow":[32],"from":[33],"social":[34],"science":[35],"methodologies":[36],"often":[37],"overlook":[38],"systematic":[39,208],"robustness":[40,209],"checks.We":[41],"identify":[42],"and":[43,89,107,130,168,203,211,216,222],"test":[44],"three":[45],"assumptions":[46],"behind":[47],"current":[48,193],"evaluation":[50,67,146,163,214],"methods:":[51],"(1)":[52],"Stability:":[53],"is":[57],"a":[58,77,115,159,169,205],"property":[59],"LLMs":[61,93,202],"rather":[62],"than":[63],"an":[64,145,179],"artifact":[65],"design,":[68],"(2)":[69],"Extrapolability:":[70],"with":[73,84],"one":[74],"culture":[75,86],"narrow":[78,166],"set":[79],"issues":[81],"predicts":[82],"others,":[88],"(3)":[90],"Steerability:":[91],"can":[94,140,174],"be":[95,148,175],"reliably":[96],"prompted":[97],"represent":[99],"specific":[100],"perspectives.Through":[102],"experiments":[103,167],"examining":[104],"both":[105],"explicit":[106],"implicit":[108],"preferences":[109],"leading":[111],"LLMs,":[112],"we":[113,156],"find":[114],"high":[116],"level":[117],"instability":[119],"presentation":[121],"formats,":[122],"incoherence":[123],"between":[124],"evaluated":[125],"versus":[126],"held-out":[127],"dimensions,":[129],"erratic":[131],"behavior":[132],"under":[133],"prompt":[134],"steering.We":[135],"show":[136],"these":[138,187],"inconsistencies":[139],"cause":[141],"results":[143,188],"very":[149],"sensitive":[150],"minor":[152],"variations":[153],"methodology.Finally,":[155],"demonstrate":[157],"case":[160],"study":[161],"design":[164],"selective":[170],"assessment":[171],"evidence":[173],"used":[176],"paint":[178],"incomplete":[180],"picture":[181],"LLMs'":[183],"properties.Overall,":[186],"highlight":[189,204],"significant":[190],"limitations":[191],"need":[206],"for":[207,213],"checks":[210],"red-teaming":[212],"results.Data":[215],"code":[217],"are":[218],"available":[219],"at":[220],"akhan02/cultural-dimension-cover-letters":[221],"ariba-k/llm-cultural-alignment-evaluation,":[223],"respectively.":[224]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":5}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}