{"id":"https://openalex.org/W7140169854","doi":"https://doi.org/10.48550/arxiv.2603.20581","title":"JUBAKU: An Adversarial Benchmark for Exposing Culturally Grounded Stereotypes in Japanese LLMs","display_name":"JUBAKU: An Adversarial Benchmark for Exposing Culturally Grounded Stereotypes in Japanese LLMs","publication_year":2026,"publication_date":"2026-03-21","ids":{"openalex":"https://openalex.org/W7140169854","doi":"https://doi.org/10.48550/arxiv.2603.20581"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2603.20581","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.20581","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2603.20581","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Shiotani, Taihei","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shiotani, Taihei","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Kaneko, Masahiro","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kaneko, Masahiro","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Niwa, Ayana","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Niwa, Ayana","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Maruyama, Yuki","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Maruyama, Yuki","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Oba, Daisuke","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Oba, Daisuke","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Ohi, Masanari","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ohi, Masanari","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":null,"display_name":"Okazaki, Naoaki","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Okazaki, Naoaki","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":7,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12262","display_name":"Hate Speech and Cyberbullying Detection","score":0.4431999921798706,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12262","display_name":"Hate Speech and Cyberbullying Detection","score":0.4431999921798706,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.13120000064373016,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12380","display_name":"Authorship Attribution and Profiling","score":0.05009999871253967,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/adversarial-system","display_name":"Adversarial system","score":0.8870000243186951},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.7024999856948853},{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.48840001225471497},{"id":"https://openalex.org/keywords/baseline","display_name":"Baseline (sea)","score":0.4839000105857849},{"id":"https://openalex.org/keywords/cultural-diversity","display_name":"Cultural diversity","score":0.45739999413490295},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.3327000141143799}],"concepts":[{"id":"https://openalex.org/C37736160","wikidata":"https://www.wikidata.org/wiki/Q1801315","display_name":"Adversarial system","level":2,"score":0.8870000243186951},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.7024999856948853},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.48840001225471497},{"id":"https://openalex.org/C12725497","wikidata":"https://www.wikidata.org/wiki/Q810247","display_name":"Baseline (sea)","level":2,"score":0.4839000105857849},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.4731000065803528},{"id":"https://openalex.org/C125209646","wikidata":"https://www.wikidata.org/wiki/Q1338878","display_name":"Cultural diversity","level":2,"score":0.45739999413490295},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.42809998989105225},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4124000072479248},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.37229999899864197},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.36980000138282776},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.3327000141143799},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3215000033378601},{"id":"https://openalex.org/C168646138","wikidata":"https://www.wikidata.org/wiki/Q11578821","display_name":"Cross-cultural","level":2,"score":0.32010000944137573},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.2840999960899353},{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.28119999170303345},{"id":"https://openalex.org/C107062074","wikidata":"https://www.wikidata.org/wiki/Q109701697","display_name":"Prejudice (legal term)","level":2,"score":0.2775999903678894},{"id":"https://openalex.org/C37773902","wikidata":"https://www.wikidata.org/wiki/Q970594","display_name":"Cultural bias","level":2,"score":0.27390000224113464},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.27059999108314514},{"id":"https://openalex.org/C2992975276","wikidata":"https://www.wikidata.org/wiki/Q11042","display_name":"Cultural background","level":4,"score":0.2533000111579895},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.25119999051094055}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2603.20581","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.20581","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2603.20581","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.20581","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/5","display_name":"Gender equality","score":0.5555011630058289}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Social":[0],"biases":[1,104,130,152],"reflected":[2],"in":[3,28,55,131,185],"language":[4,30],"are":[5],"inherently":[6],"shaped":[7],"by":[8,118],"cultural":[9,50,95,108],"norms,":[10,51],"which":[11],"vary":[12],"significantly":[13],"across":[14,105],"regions":[15],"and":[16,126,141,192],"lead":[17],"to":[18,47,66,93,101,124,171,196],"diverse":[19],"manifestations":[20],"of":[21,25,41,160,166],"stereotypes.":[22],"Existing":[23],"evaluations":[24],"social":[26,129],"bias":[27],"large":[29],"models":[31,149],"(LLMs)":[32],"for":[33],"non-English":[34],"contexts,":[35],"however,":[36],"often":[37],"rely":[38],"on":[39,139,153,176],"translations":[40],"English":[42,146],"benchmarks.":[43,147,179],"Such":[44],"benchmarks":[45,60],"fail":[46],"reflect":[48],"local":[49],"including":[52],"those":[53],"found":[54],"Japanese.":[56],"For":[57],"instance,":[58],"Western":[59],"may":[61],"overlook":[62],"Japan-specific":[63],"stereotypes":[64],"related":[65],"hierarchical":[67],"relationships,":[68],"regional":[69],"dialects,":[70],"or":[71],"traditional":[72],"gender":[73],"roles.":[74],"To":[75],"address":[76],"this":[77],"limitation,":[78],"we":[79],"introduce":[80],"Japanese":[81,94,120,132,137],"cUlture":[82],"adversarial":[83,99,194],"BiAs":[84],"benchmarK":[85],"Under":[86],"handcrafted":[87],"creation":[88],"(JUBAKU),":[89],"a":[90],"benchmark":[91],"tailored":[92],"contexts.":[96],"JUBAKU":[97,113,140],"uses":[98],"construction":[100],"expose":[102],"latent":[103,128],"ten":[106],"distinct":[107],"categories.":[109],"Unlike":[110],"existing":[111],"benchmarks,":[112],"features":[114],"dialogue":[115],"scenarios":[116],"hand-crafted":[117],"native":[119],"annotators,":[121],"specifically":[122],"designed":[123],"trigger":[125],"reveal":[127],"LLMs.":[133,197],"We":[134],"evaluated":[135],"nine":[136],"LLMs":[138],"three":[142],"others":[143],"adapted":[144],"from":[145,169],"All":[148],"clearly":[150],"exhibited":[151],"JUBAKU,":[154],"performing":[155],"below":[156],"the":[157,177],"random":[158],"baseline":[159],"50%":[161],"with":[162],"an":[163],"average":[164],"accuracy":[165,175,184],"23%":[167],"(ranging":[168],"13%":[170],"33%),":[172],"despite":[173],"higher":[174],"other":[178],"Human":[180],"annotators":[181],"achieved":[182],"91%":[183],"identifying":[186],"unbiased":[187],"responses,":[188],"confirming":[189],"JUBAKU's":[190],"reliability":[191],"its":[193],"nature":[195]},"counts_by_year":[],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2026-03-25T00:00:00"}