{"id":"https://openalex.org/W4412876880","doi":"https://doi.org/10.1145/3711896.3737375","title":"CityBench: Evaluating the Capabilities of Large Language Models for Urban Tasks","display_name":"CityBench: Evaluating the Capabilities of Large Language Models for Urban Tasks","publication_year":2025,"publication_date":"2025-08-03","ids":{"openalex":"https://openalex.org/W4412876880","doi":"https://doi.org/10.1145/3711896.3737375"},"language":"en","primary_location":{"id":"doi:10.1145/3711896.3737375","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3711896.3737375","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3711896.3737375","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 31st ACM SIGKDD Conference on Knowledge Discovery and Data Mining V.2","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3711896.3737375","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100668035","display_name":"Jie Feng","orcid":"https://orcid.org/0000-0003-3279-7117"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Jie Feng","raw_affiliation_strings":["Department of Electronic Engineering, BNRist, Tsinghua University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0003-3279-7117","affiliations":[{"raw_affiliation_string":"Department of Electronic Engineering, BNRist, Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5061284465","display_name":"Jun Zhang","orcid":"https://orcid.org/0000-0003-1232-3475"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jun Zhang","raw_affiliation_strings":["Department of Electronic Engineering, BNRist, Tsinghua University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0003-1232-3475","affiliations":[{"raw_affiliation_string":"Department of Electronic Engineering, BNRist, Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5069964817","display_name":"T. Liu","orcid":"https://orcid.org/0009-0005-9729-9700"},"institutions":[{"id":"https://openalex.org/I21193070","display_name":"Beijing Jiaotong University","ror":"https://ror.org/01yj56c84","country_code":"CN","type":"education","lineage":["https://openalex.org/I21193070"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Tianhui Liu","raw_affiliation_strings":["School of Electronic and Information Engineering, Beijing Jiaotong University, Beijing, China"],"raw_orcid":"https://orcid.org/0009-0005-9729-9700","affiliations":[{"raw_affiliation_string":"School of Electronic and Information Engineering, Beijing Jiaotong University, Beijing, China","institution_ids":["https://openalex.org/I21193070"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5042093698","display_name":"Xin Zhang","orcid":"https://orcid.org/0000-0002-2506-7370"},"institutions":[{"id":"https://openalex.org/I3131625388","display_name":"University Town of Shenzhen","ror":"https://ror.org/05f5j6225","country_code":"CN","type":"education","lineage":["https://openalex.org/I3131625388"]},{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xin Zhang","raw_affiliation_strings":["Shenzhen International Graduate School, Tsinghua University, Shenzhen, China"],"raw_orcid":"https://orcid.org/0000-0002-2506-7370","affiliations":[{"raw_affiliation_string":"Shenzhen International Graduate School, Tsinghua University, Shenzhen, China","institution_ids":["https://openalex.org/I3131625388","https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5097902336","display_name":"Tianjian Ouyang","orcid":"https://orcid.org/0009-0007-4881-3165"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Tianjian Ouyang","raw_affiliation_strings":["Department of Electronic Engineering, BNRist, Tsinghua University, Beijing, China"],"raw_orcid":"https://orcid.org/0009-0007-4881-3165","affiliations":[{"raw_affiliation_string":"Department of Electronic Engineering, BNRist, Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5107903855","display_name":"J. Yan","orcid":null},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Junbo Yan","raw_affiliation_strings":["Department of Electronic Engineering, BNRist, Tsinghua University, Beijing, China"],"raw_orcid":"https://orcid.org/0009-0000-7135-631X","affiliations":[{"raw_affiliation_string":"Department of Electronic Engineering, BNRist, Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5069959549","display_name":"Yangzhou Du","orcid":null},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yuwei Du","raw_affiliation_strings":["Department of Electronic Engineering, BNRist, Tsinghua University, Beijing, China"],"raw_orcid":"https://orcid.org/0009-0009-7197-4367","affiliations":[{"raw_affiliation_string":"Department of Electronic Engineering, BNRist, Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101384209","display_name":"Siqi Guo","orcid":null},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Siqi Guo","raw_affiliation_strings":["Department of Electronic Engineering, Tsinghua University, Beijing, China"],"raw_orcid":"https://orcid.org/0009-0009-0837-7498","affiliations":[{"raw_affiliation_string":"Department of Electronic Engineering, Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100355277","display_name":"Yong Li","orcid":"https://orcid.org/0000-0001-5617-1659"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yong Li","raw_affiliation_strings":["Department of Electronic Engineering, BNRist, Tsinghua University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0001-5617-1659","affiliations":[{"raw_affiliation_string":"Department of Electronic Engineering, BNRist, Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":9,"corresponding_author_ids":["https://openalex.org/A5100668035"],"corresponding_institution_ids":["https://openalex.org/I99065089"],"apc_list":null,"apc_paid":null,"fwci":11.1279,"has_fulltext":true,"cited_by_count":6,"citation_normalized_percentile":{"value":0.98253925,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":97,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"5413","last_page":"5424"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11980","display_name":"Human Mobility and Location-Based Analysis","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/3313","display_name":"Transportation"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T11980","display_name":"Human Mobility and Location-Based Analysis","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/3313","display_name":"Transportation"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11479","display_name":"Smart Cities and Technologies","score":0.9891999959945679,"subfield":{"id":"https://openalex.org/subfields/2214","display_name":"Media Technology"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10298","display_name":"Urban Transport and Accessibility","score":0.9746000170707703,"subfield":{"id":"https://openalex.org/subfields/3313","display_name":"Transportation"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7665224075317383},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.33665919303894043}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7665224075317383},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.33665919303894043}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3711896.3737375","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3711896.3737375","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3711896.3737375","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 31st ACM SIGKDD Conference on Knowledge Discovery and Data Mining V.2","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3711896.3737375","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3711896.3737375","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3711896.3737375","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 31st ACM SIGKDD Conference on Knowledge Discovery and Data Mining V.2","raw_type":"proceedings-article"},"sustainable_development_goals":[{"display_name":"Sustainable cities and communities","id":"https://metadata.un.org/sdg/11","score":0.550000011920929}],"awards":[{"id":"https://openalex.org/G3614744892","display_name":null,"funder_award_id":"U23B2030","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320321543","display_name":"China Postdoctoral Science Foundation","ror":"https://ror.org/0426zh255"},{"id":"https://openalex.org/F4320322392","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549"},{"id":"https://openalex.org/F4320335777","display_name":"National Key Research and Development Program of China","ror":null}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4412876880.pdf","grobid_xml":"https://content.openalex.org/works/W4412876880.grobid-xml"},"referenced_works_count":28,"referenced_works":["https://openalex.org/W651043869","https://openalex.org/W1969085745","https://openalex.org/W2056877664","https://openalex.org/W2112738128","https://openalex.org/W2329660289","https://openalex.org/W2513506629","https://openalex.org/W2799002257","https://openalex.org/W2903172725","https://openalex.org/W2979727876","https://openalex.org/W2998167534","https://openalex.org/W4298263076","https://openalex.org/W4312795296","https://openalex.org/W4328127395","https://openalex.org/W4382138521","https://openalex.org/W4387321091","https://openalex.org/W4388834509","https://openalex.org/W4390100400","https://openalex.org/W4393065402","https://openalex.org/W4394938913","https://openalex.org/W4401857326","https://openalex.org/W4402671260","https://openalex.org/W4403791888","https://openalex.org/W4404654659","https://openalex.org/W4411119500","https://openalex.org/W4411735734","https://openalex.org/W6600165366","https://openalex.org/W6600281463","https://openalex.org/W6604047254"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W4391913857","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W4396696052"],"abstract_inverted_index":{"As":[0],"large":[1],"language":[2],"models":[3,22],"(LLMs)":[4],"continue":[5],"to":[6,27,124,132,208,242],"advance":[7,243],"and":[8,14,20,32,52,82,130,140,153,165,178,190,199,217,224,237,246],"gain":[9],"widespread":[10],"use,":[11],"establishing":[12],"systematic":[13,51,64,106],"reliable":[15],"evaluation":[16,54,65,101],"methodologies":[17],"for":[18,45,67,108,114,233],"LLMs":[19,44,113,164,177,241],"vision-language":[21],"(VLMs)":[23],"has":[24],"become":[25],"essential":[26],"ensure":[28],"their":[29],"real-world":[30],"effectiveness":[31],"reliability.":[33],"There":[34],"have":[35],"been":[36],"some":[37],"early":[38],"explorations":[39],"about":[40],"the":[41,72,77,83,88,104,110,126,156,171,196,210,234,249],"usability":[42],"of":[43,74,79,87,112,151,202,240],"limited":[46],"urban":[47,68,75,89,118,128,135,146,186,203,212],"tasks,":[48],"but":[49],"a":[50,63],"scalable":[53],"benchmark":[55,66,107],"is":[56],"still":[57],"lacking.":[58],"The":[59],"challenge":[60],"in":[61,71,117,148,167,184,248],"constructing":[62],"research":[69,247],"lies":[70],"diversity":[73],"data,":[76],"complexity":[78],"application":[80],"scenarios":[81],"highly":[84],"dynamic":[85],"nature":[86],"environment.":[90],"In":[91],"this":[92],"paper,":[93],"we":[94,121,142,173],"design":[95,143],"CityBench,":[96],"an":[97],"interactive":[98],"simulator":[99],"based":[100],"platform,":[102],"as":[103,155],"first":[105],"evaluating":[109],"capabilities":[111],"diverse":[115,127,185],"tasks":[116,147,187,213,245],"research.":[119],"First,":[120],"build":[122],"CityData":[123,139],"integrate":[125],"data":[129],"CitySimu":[131],"simulate":[133],"fine-grained":[134],"dynamics.":[136],"Based":[137],"on":[138],"CitySimu,":[141],"8":[144],"representative":[145],"2":[149],"categories":[150],"perception-understanding":[152],"decision-making":[154],"CityBench.":[157],"With":[158],"extensive":[159],"results":[160],"from":[161],"30":[162],"well-known":[163],"VLMs":[166,179],"13":[168],"cities":[169],"around":[170],"world,":[172],"find":[174],"that":[175],"advanced":[176],"can":[180],"achieve":[181],"competitive":[182],"performance":[183],"requiring":[188,214],"commonsense":[189],"semantic":[191,200],"understanding":[192,195],"abilities,":[193,220],"e.g.,":[194,221],"human":[197],"dynamics":[198],"inference":[201],"images.":[204],"Meanwhile,":[205],"they":[206],"fail":[207],"solve":[209],"challenging":[211],"professional":[215],"knowledge":[216],"high-level":[218],"numerical":[219],"geospatial":[222],"prediction":[223],"traffic":[225],"control":[226],"task.":[227],"These":[228],"findings":[229],"provide":[230],"critical":[231],"insights":[232],"effective":[235],"utilization":[236],"further":[238],"development":[239],"urban-related":[244],"future.":[250]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":4}],"updated_date":"2026-05-05T08:41:31.759640","created_date":"2025-10-10T00:00:00"}