{"id":"https://openalex.org/W4399364305","doi":"https://doi.org/10.1145/3630106.3658967","title":"WorldBench: Quantifying Geographic Disparities in LLM Factual Recall","display_name":"WorldBench: Quantifying Geographic Disparities in LLM Factual Recall","publication_year":2024,"publication_date":"2024-06-03","ids":{"openalex":"https://openalex.org/W4399364305","doi":"https://doi.org/10.1145/3630106.3658967"},"language":"en","primary_location":{"id":"doi:10.1145/3630106.3658967","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3630106.3658967","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3630106.3658967","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"The 2024 ACM Conference on Fairness, Accountability, and Transparency","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3630106.3658967","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5077078550","display_name":"Mazda Moayeri","orcid":"https://orcid.org/0009-0006-2842-5352"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mazda Moayeri","raw_affiliation_strings":["University of Maryland, United States of America"],"raw_orcid":"https://orcid.org/0009-0006-2842-5352","affiliations":[{"raw_affiliation_string":"University of Maryland, United States of America","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5074699094","display_name":"Elham Tabassi","orcid":"https://orcid.org/0000-0001-9536-2722"},"institutions":[{"id":"https://openalex.org/I87216513","display_name":"Michigan State University","ror":"https://ror.org/05hs6h993","country_code":"US","type":"education","lineage":["https://openalex.org/I87216513"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Elham Tabassi","raw_affiliation_strings":["Michigan State University, USA"],"raw_orcid":"https://orcid.org/0000-0001-9536-2722","affiliations":[{"raw_affiliation_string":"Michigan State University, USA","institution_ids":["https://openalex.org/I87216513"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5025450606","display_name":"Soheil Feizi","orcid":"https://orcid.org/0000-0003-0944-8242"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Soheil Feizi","raw_affiliation_strings":["University of Maryland, United States of America"],"raw_orcid":"https://orcid.org/0000-0003-0944-8242","affiliations":[{"raw_affiliation_string":"University of Maryland, United States of America","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":22,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1211","last_page":"1228"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T14509","display_name":"demographic modeling and climate adaptation","score":0.8166999816894531,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T14509","display_name":"demographic modeling and climate adaptation","score":0.8166999816894531,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.8148000240325928,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11052","display_name":"Energy Load and Power Forecasting","score":0.699999988079071,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/popularity","display_name":"Popularity","score":0.6745893359184265},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.6255176067352295},{"id":"https://openalex.org/keywords/footprint","display_name":"Footprint","score":0.5655940175056458},{"id":"https://openalex.org/keywords/recall","display_name":"Recall","score":0.5574844479560852},{"id":"https://openalex.org/keywords/population","display_name":"Population","score":0.5259297490119934},{"id":"https://openalex.org/keywords/citation","display_name":"Citation","score":0.45065897703170776},{"id":"https://openalex.org/keywords/geography","display_name":"Geography","score":0.4308415651321411},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.33451342582702637},{"id":"https://openalex.org/keywords/political-science","display_name":"Political science","score":0.272086501121521},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.22526180744171143},{"id":"https://openalex.org/keywords/demography","display_name":"Demography","score":0.20685040950775146},{"id":"https://openalex.org/keywords/cartography","display_name":"Cartography","score":0.16901519894599915},{"id":"https://openalex.org/keywords/cognitive-psychology","display_name":"Cognitive psychology","score":0.1642686128616333},{"id":"https://openalex.org/keywords/sociology","display_name":"Sociology","score":0.14580705761909485},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.10756862163543701},{"id":"https://openalex.org/keywords/law","display_name":"Law","score":0.08596929907798767}],"concepts":[{"id":"https://openalex.org/C2780586970","wikidata":"https://www.wikidata.org/wiki/Q1357284","display_name":"Popularity","level":2,"score":0.6745893359184265},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.6255176067352295},{"id":"https://openalex.org/C132943942","wikidata":"https://www.wikidata.org/wiki/Q2562511","display_name":"Footprint","level":2,"score":0.5655940175056458},{"id":"https://openalex.org/C100660578","wikidata":"https://www.wikidata.org/wiki/Q18733","display_name":"Recall","level":2,"score":0.5574844479560852},{"id":"https://openalex.org/C2908647359","wikidata":"https://www.wikidata.org/wiki/Q2625603","display_name":"Population","level":2,"score":0.5259297490119934},{"id":"https://openalex.org/C2778805511","wikidata":"https://www.wikidata.org/wiki/Q1713","display_name":"Citation","level":2,"score":0.45065897703170776},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.4308415651321411},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.33451342582702637},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.272086501121521},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.22526180744171143},{"id":"https://openalex.org/C149923435","wikidata":"https://www.wikidata.org/wiki/Q37732","display_name":"Demography","level":1,"score":0.20685040950775146},{"id":"https://openalex.org/C58640448","wikidata":"https://www.wikidata.org/wiki/Q42515","display_name":"Cartography","level":1,"score":0.16901519894599915},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.1642686128616333},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.14580705761909485},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.10756862163543701},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.08596929907798767},{"id":"https://openalex.org/C166957645","wikidata":"https://www.wikidata.org/wiki/Q23498","display_name":"Archaeology","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3630106.3658967","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3630106.3658967","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3630106.3658967","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"The 2024 ACM Conference on Fairness, Accountability, and Transparency","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3630106.3658967","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3630106.3658967","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3630106.3658967","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"The 2024 ACM Conference on Fairness, Accountability, and Transparency","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":false},"content_urls":{"pdf":"https://content.openalex.org/works/W4399364305.pdf"},"referenced_works_count":4,"referenced_works":["https://openalex.org/W2979826702","https://openalex.org/W4327519588","https://openalex.org/W4385570777","https://openalex.org/W4385572316"],"related_works":["https://openalex.org/W2368605798","https://openalex.org/W2518037665","https://openalex.org/W2348524959","https://openalex.org/W2477036161","https://openalex.org/W2368049389","https://openalex.org/W2384861574","https://openalex.org/W4294565801","https://openalex.org/W2170801710","https://openalex.org/W2952704802","https://openalex.org/W2741781807"],"abstract_inverted_index":{"As":[0],"large":[1],"language":[2],"models":[3,15,29,156],"(LLMs)":[4],"continue":[5],"to":[6,16,36,90,118,126,169,177,189],"improve":[7],"and":[8,60,80,88,101,132,166,195],"gain":[9],"popularity,":[10],"some":[11],"may":[12],"use":[13],"the":[14,46,68,77,158,197],"recall":[17],"facts,":[18],"despite":[19],"well":[20],"documented":[21],"limitations":[22],"with":[23],"LLM":[24,45],"factuality.":[25],"Towards":[26],"ensuring":[27],"that":[28],"work":[30],"reliably":[31],"for":[32,112],"all,":[33],"we":[34,55,94],"seek":[35],"uncover":[37],"if":[38],"geographic":[39,190],"disparities":[40,125,191],"emerge":[41],"when":[42,171],"asking":[43],"an":[44,172],"same":[47],"question":[48],"about":[49],"different":[50],"countries.":[51,121],"To":[52],"this":[53],"end,":[54],"present":[56],"WorldBench,":[57],"a":[58,92,167],"dynamic":[59],"flexible":[61],"benchmark":[62,185],"composed":[63],"of":[64,76,152,180,199],"per-country":[65],"data":[66],"from":[67,114],"World":[69,135,159],"Bank.":[70],"In":[71],"extensive":[72],"experiments":[73],"on":[74,99],"state":[75],"art":[78],"open":[79],"closed":[81],"source":[82],"models,":[83],"including":[84],"GPT-4,":[85],"Gemini,":[86],"Llama-2,":[87],"Vicuna,":[89],"name":[91],"few,":[93],"find":[95],"significant":[96],"biases":[97],"based":[98],"region":[100],"income":[102],"level.":[103],"For":[104],"example,":[105],"error":[106],"rates":[107],"are":[108],"1.5":[109],"times":[110],"higher":[111],"countries":[113],"Sub-Saharan":[115],"Africa":[116],"compared":[117],"North":[119],"American":[120],"We":[122,182],"observe":[123],"these":[124,200],"be":[127],"consistent":[128],"over":[129],"20":[130],"LLMs":[131,194],"11":[133],"individual":[134],"Bank":[136,160],"indicators":[137],"(i.e.":[138],"specific":[139],"statistics,":[140,165],"such":[141],"as":[142],"population":[143],"or":[144],"CO2":[145],"emissions).":[146],"WorldBench":[147],"also":[148],"enables":[149],"automatic":[150],"detection":[151],"citation":[153],"hallucination,":[154],"where":[155],"cite":[157],"itself":[161],"while":[162],"providing":[163],"false":[164],"manner":[168],"assess":[170],"LLM\u2019s":[173],"stored":[174],"facts":[175],"begin":[176],"go":[178],"out":[179],"date.":[181],"hope":[183],"our":[184],"will":[186],"draw":[187],"attention":[188],"in":[192],"existing":[193],"facilitate":[196],"remedying":[198],"biases.":[201]},"counts_by_year":[{"year":2026,"cited_by_count":6},{"year":2025,"cited_by_count":14},{"year":2024,"cited_by_count":2}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}