{"id":"https://openalex.org/W4404481617","doi":"https://doi.org/10.1145/3687123.3698286","title":"Evaluation of Code LLMs on Geospatial Code Generation","display_name":"Evaluation of Code LLMs on Geospatial Code Generation","publication_year":2024,"publication_date":"2024-10-29","ids":{"openalex":"https://openalex.org/W4404481617","doi":"https://doi.org/10.1145/3687123.3698286"},"language":"en","primary_location":{"id":"doi:10.1145/3687123.3698286","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3687123.3698286","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 7th ACM SIGSPATIAL International Workshop on AI for Geographic Knowledge Discovery","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3687123.3698286","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5024393632","display_name":"Piotr Gramacki","orcid":"https://orcid.org/0000-0002-4587-5586"},"institutions":[{"id":"https://openalex.org/I11923345","display_name":"Wroc\u0142aw University of Science and Technology","ror":"https://ror.org/008fyn775","country_code":"PL","type":"education","lineage":["https://openalex.org/I11923345"]},{"id":"https://openalex.org/I686019","display_name":"AGH University of Krakow","ror":"https://ror.org/00bas1c41","country_code":"PL","type":"education","lineage":["https://openalex.org/I686019"]}],"countries":["PL"],"is_corresponding":false,"raw_author_name":"Piotr Gramacki","raw_affiliation_strings":["Wroc\u0142aw University of Science and Technology, Department of Artificial Intelligence Kraina.AI, Wroc\u0142aw, Poland","- ( ) of ,"],"raw_orcid":"https://orcid.org/0000-0002-4587-5586","affiliations":[{"raw_affiliation_string":"Wroc\u0142aw University of Science and Technology, Department of Artificial Intelligence Kraina.AI, Wroc\u0142aw, Poland","institution_ids":["https://openalex.org/I11923345","https://openalex.org/I686019"]},{"raw_affiliation_string":"- ( ) of ,","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5055101594","display_name":"Bruno Martins","orcid":"https://orcid.org/0000-0002-3856-2936"},"institutions":[{"id":"https://openalex.org/I121345201","display_name":"Instituto de Engenharia de Sistemas e Computadores Investiga\u00e7\u00e3o e Desenvolvimento","ror":"https://ror.org/04mqy3p58","country_code":"PT","type":"nonprofit","lineage":["https://openalex.org/I121345201","https://openalex.org/I4210125590"]},{"id":"https://openalex.org/I141596103","display_name":"University of Lisbon","ror":"https://ror.org/01c27hj86","country_code":"PT","type":"education","lineage":["https://openalex.org/I141596103"]},{"id":"https://openalex.org/I4210164862","display_name":"Artificial Intelligence in Medicine (Canada)","ror":"https://ror.org/05p590m36","country_code":"CA","type":"company","lineage":["https://openalex.org/I4210164862"]}],"countries":["CA","PT"],"is_corresponding":false,"raw_author_name":"Bruno Martins","raw_affiliation_strings":["INESC-ID & LUMLIS (Lisbon ELLIS Unit), Instituto Superior T\u00e9cnico, University of Lisbon, Lisbon, Portugal","of and Technology of Artificial Intelligence. ,","- ( ) of ,"],"raw_orcid":"https://orcid.org/0000-0002-3856-2936","affiliations":[{"raw_affiliation_string":"INESC-ID & LUMLIS (Lisbon ELLIS Unit), Instituto Superior T\u00e9cnico, University of Lisbon, Lisbon, Portugal","institution_ids":["https://openalex.org/I121345201","https://openalex.org/I141596103"]},{"raw_affiliation_string":"of and Technology of Artificial Intelligence. ,","institution_ids":["https://openalex.org/I4210164862"]},{"raw_affiliation_string":"- ( ) of ,","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5029267128","display_name":"Piotr Szyma\u0144ski","orcid":"https://orcid.org/0000-0002-7733-3239"},"institutions":[{"id":"https://openalex.org/I11923345","display_name":"Wroc\u0142aw University of Science and Technology","ror":"https://ror.org/008fyn775","country_code":"PL","type":"education","lineage":["https://openalex.org/I11923345"]},{"id":"https://openalex.org/I686019","display_name":"AGH University of Krakow","ror":"https://ror.org/00bas1c41","country_code":"PL","type":"education","lineage":["https://openalex.org/I686019"]}],"countries":["PL"],"is_corresponding":false,"raw_author_name":"Piotr Szyma\u0144ski","raw_affiliation_strings":["Wroc\u0142aw University of Science and Technology, Department of Artificial Intelligence Kraina.AI, Wroc\u0142aw, Poland","- ( ) of ,"],"raw_orcid":"https://orcid.org/0000-0002-7733-3239","affiliations":[{"raw_affiliation_string":"Wroc\u0142aw University of Science and Technology, Department of Artificial Intelligence Kraina.AI, Wroc\u0142aw, Poland","institution_ids":["https://openalex.org/I11923345","https://openalex.org/I686019"]},{"raw_affiliation_string":"- ( ) of ,","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":5,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":25,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"54","last_page":"62"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10215","display_name":"Semantic Web and Ontologies","score":0.9973000288009644,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10215","display_name":"Semantic Web and Ontologies","score":0.9973000288009644,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.989799976348877,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11710","display_name":"Biomedical Text Mining and Ontologies","score":0.965499997138977,"subfield":{"id":"https://openalex.org/subfields/1312","display_name":"Molecular Biology"},"field":{"id":"https://openalex.org/fields/13","display_name":"Biochemistry, Genetics and Molecular Biology"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/geospatial-analysis","display_name":"Geospatial analysis","score":0.8336423635482788},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6755215525627136},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.6281354427337646},{"id":"https://openalex.org/keywords/computer-security","display_name":"Computer security","score":0.3655027151107788},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.331190288066864},{"id":"https://openalex.org/keywords/database","display_name":"Database","score":0.3210963010787964},{"id":"https://openalex.org/keywords/geography","display_name":"Geography","score":0.17461082339286804},{"id":"https://openalex.org/keywords/remote-sensing","display_name":"Remote sensing","score":0.12995001673698425}],"concepts":[{"id":"https://openalex.org/C9770341","wikidata":"https://www.wikidata.org/wiki/Q1938983","display_name":"Geospatial analysis","level":2,"score":0.8336423635482788},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6755215525627136},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.6281354427337646},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.3655027151107788},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.331190288066864},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.3210963010787964},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.17461082339286804},{"id":"https://openalex.org/C62649853","wikidata":"https://www.wikidata.org/wiki/Q199687","display_name":"Remote sensing","level":1,"score":0.12995001673698425},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3687123.3698286","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3687123.3698286","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 7th ACM SIGSPATIAL International Workshop on AI for Geographic Knowledge Discovery","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3687123.3698286","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3687123.3698286","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 7th ACM SIGSPATIAL International Workshop on AI for Geographic Knowledge Discovery","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":11,"referenced_works":["https://openalex.org/W2101105183","https://openalex.org/W2560674852","https://openalex.org/W3011565817","https://openalex.org/W4285600327","https://openalex.org/W4388616070","https://openalex.org/W4388834509","https://openalex.org/W4390100400","https://openalex.org/W4393002303","https://openalex.org/W4398141811","https://openalex.org/W4399730268","https://openalex.org/W6930223383"],"related_works":["https://openalex.org/W4367313141","https://openalex.org/W2004086023","https://openalex.org/W2733999579","https://openalex.org/W2110217573","https://openalex.org/W4283374591","https://openalex.org/W2910751785","https://openalex.org/W4390100400","https://openalex.org/W4366547507","https://openalex.org/W4362512700","https://openalex.org/W2074396925"],"abstract_inverted_index":{"Software":[0],"development":[1,237],"support":[2],"tools":[3,146],"have":[4],"been":[5],"studied":[6],"for":[7,19,28,38,56,105,159,182,195,224,259],"a":[8,54,62,111,130,167,188,211],"long":[9],"time,":[10],"with":[11,69,132,246],"recent":[12],"approaches":[13],"using":[14],"Large":[15],"Language":[16],"Models":[17],"(LLMs)":[18],"code":[20,27,67,86,106,181,192,196,209],"generation.":[21],"These":[22,249],"models":[23,239,250],"can":[24,50,71,218],"generate":[25],"Python":[26],"data":[29,77,142],"science":[30],"and":[31,60,85,124,144,206],"machine":[32],"learning":[33,64],"applications.":[34,261],"LLMs":[35,70,88,194,226],"are":[36,89],"helpful":[37],"software":[39,58],"engineers":[40],"because":[41],"they":[42],"increase":[43],"productivity":[44],"in":[45,75,138,198,227],"daily":[46],"work.":[47],"An":[48],"LLM":[49],"also":[51,72],"serve":[52,219],"as":[53,220],"\"mentor\"":[55],"inexperienced":[57],"developers,":[59],"be":[61,73],"viable":[63],"support.":[65],"High-quality":[66],"generation":[68,87,107,193,197],"beneficial":[74],"geospatial":[76,94,114,118,145,200,243,260],"science.":[78],"However,":[79],"this":[80,217],"domain":[81],"poses":[82],"different":[83],"challenges,":[84],"typically":[90],"not":[91],"evaluated":[92],"on":[93,110,121,210],"tasks.":[95,115],"Here,":[96],"we":[97,100,128,165,186],"show":[98],"how":[99],"constructed":[101],"an":[102,221],"evaluation":[103,208,222],"benchmark":[104,223],"models,":[108],"based":[109,120],"selection":[112,189],"of":[113,151,169,190,241,255],"We":[116,202],"categorised":[117],"tasks":[119,133,245],"their":[122],"complexity":[123],"required":[125],"tools.":[126],"Then,":[127],"created":[129,158],"dataset":[131,149,205,231],"that":[134,155,172,216],"test":[135,170],"model":[136],"capabilities":[137],"spatial":[139,141],"reasoning,":[140],"processing,":[143],"usage.":[147],"The":[148],"consists":[150],"specific":[152],"coding":[153,244,256],"problems":[154],"were":[156],"manually":[157],"high":[160,247],"quality.":[161],"For":[162],"every":[163],"problem,":[164],"proposed":[166],"set":[168],"scenarios":[171],"make":[173],"it":[174],"possible":[175],"to":[176,235],"automatically":[177],"check":[178],"the":[179,199,228,236,253],"generated":[180],"correctness.":[183],"In":[184],"addition,":[185],"tested":[187],"existing":[191],"domain.":[201],"share":[203],"our":[204],"reproducible":[207],"public":[212],"GitHub":[213],"repository1,":[214],"arguing":[215],"new":[225,238],"future.":[229],"Our":[230],"will":[232,251],"hopefully":[233],"contribute":[234],"capable":[240],"solving":[242],"accuracy.":[248],"enable":[252],"creation":[254],"assistants":[257],"tailored":[258]},"counts_by_year":[{"year":2026,"cited_by_count":9},{"year":2025,"cited_by_count":16}],"updated_date":"2026-07-26T07:53:14.480251","created_date":"2025-10-10T00:00:00"}