{"id":"https://openalex.org/W7163994139","doi":"https://doi.org/10.48550/arxiv.2606.08036","title":"GIScholarBench: Benchmarking LLM Overconfidence in GIS Research","display_name":"GIScholarBench: Benchmarking LLM Overconfidence in GIS Research","publication_year":2026,"publication_date":"2026-06-06","ids":{"openalex":"https://openalex.org/W7163994139","doi":"https://doi.org/10.48550/arxiv.2606.08036"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.08036","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.08036","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.08036","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5138203053","display_name":"Zongrng Li","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Zongrng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138208519","display_name":"Mingzheng Yang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yang, Mingzheng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5033785339","display_name":"Lei Zou","orcid":"https://orcid.org/0000-0002-8586-4400"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zou, Lei","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138202285","display_name":"Hongxu Ma","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ma, Hongxu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138209525","display_name":"Hao Tian","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tian, Hao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138252284","display_name":"Siqi Zhou","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhou, Siqi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138237800","display_name":"Wenjing Gong","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gong, Wenjing","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138273421","display_name":"Kaili Zhang (709422)","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Kaili","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138282687","display_name":"Bingqian Chen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Bingqian","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138256666","display_name":"Mitch Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Mitch","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5138204478","display_name":"Yifan Yang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yang, Yifan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10757","display_name":"Geographic Information Systems Studies","score":0.30070000886917114,"subfield":{"id":"https://openalex.org/subfields/3305","display_name":"Geography, Planning and Development"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10757","display_name":"Geographic Information Systems Studies","score":0.30070000886917114,"subfield":{"id":"https://openalex.org/subfields/3305","display_name":"Geography, Planning and Development"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T13910","display_name":"Computational and Text Analysis Methods","score":0.14079999923706055,"subfield":{"id":"https://openalex.org/subfields/3300","display_name":"General Social Sciences"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11819","display_name":"Data-Driven Disease Surveillance","score":0.12139999866485596,"subfield":{"id":"https://openalex.org/subfields/2713","display_name":"Epidemiology"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/overconfidence-effect","display_name":"Overconfidence effect","score":0.9085999727249146},{"id":"https://openalex.org/keywords/citation","display_name":"Citation","score":0.5583000183105469},{"id":"https://openalex.org/keywords/benchmarking","display_name":"Benchmarking","score":0.5504000186920166},{"id":"https://openalex.org/keywords/metadata","display_name":"Metadata","score":0.5494999885559082},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.5015000104904175},{"id":"https://openalex.org/keywords/ranking","display_name":"Ranking (information retrieval)","score":0.4325999915599823}],"concepts":[{"id":"https://openalex.org/C51110983","wikidata":"https://www.wikidata.org/wiki/Q16503490","display_name":"Overconfidence effect","level":2,"score":0.9085999727249146},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5839999914169312},{"id":"https://openalex.org/C2778805511","wikidata":"https://www.wikidata.org/wiki/Q1713","display_name":"Citation","level":2,"score":0.5583000183105469},{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.5504000186920166},{"id":"https://openalex.org/C93518851","wikidata":"https://www.wikidata.org/wiki/Q180160","display_name":"Metadata","level":2,"score":0.5494999885559082},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.5015000104904175},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.47540000081062317},{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.4325999915599823},{"id":"https://openalex.org/C183115368","wikidata":"https://www.wikidata.org/wiki/Q856577","display_name":"Weighting","level":2,"score":0.3993000090122223},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.38609999418258667},{"id":"https://openalex.org/C41458344","wikidata":"https://www.wikidata.org/wiki/Q732577","display_name":"Publication","level":2,"score":0.35850000381469727},{"id":"https://openalex.org/C105345328","wikidata":"https://www.wikidata.org/wiki/Q206276","display_name":"Citation analysis","level":3,"score":0.3386000096797943},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.33489999175071716},{"id":"https://openalex.org/C100776233","wikidata":"https://www.wikidata.org/wiki/Q2532492","display_name":"Bridge (graph theory)","level":2,"score":0.3086000084877014},{"id":"https://openalex.org/C17231256","wikidata":"https://www.wikidata.org/wiki/Q5156540","display_name":"Completeness (order theory)","level":2,"score":0.2913999855518341},{"id":"https://openalex.org/C82261069","wikidata":"https://www.wikidata.org/wiki/Q1925365","display_name":"Yardstick","level":2,"score":0.2913999855518341},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.2808000147342682},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.25929999351501465}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.08036","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.08036","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.08036","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.08036","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4","score":0.6569735407829285}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Large":[0],"language":[1],"models":[2,137,160],"(LLMs)":[3],"are":[4,146,175],"increasingly":[5],"used":[6],"in":[7,74,217,222,227],"academic":[8],"research":[9,97,182,231],"workflows,":[10],"but":[11,135,158,211],"scholarly":[12],"tasks":[13,87],"require":[14],"high":[15],"factual":[16,215],"precision":[17],"and":[18,37,58,81,96,107,142,168,195,225],"therefore":[19],"expose":[20],"a":[21,52,67,162],"key":[22],"weakness:":[23],"overconfidence.":[24],"Here,":[25],"overconfidence":[26,122,208,226],"is":[27,45,209],"defined":[28],"behaviorally":[29],"as":[30,51],"the":[31,42,132,155],"tendency":[32],"to":[33],"produce":[34],"confident,":[35],"assertive,":[36],"well-formatted":[38],"outputs":[39],"even":[40],"when":[41,144],"underlying":[43],"knowledge":[44],"incomplete":[46],"or":[47],"unverifiable,":[48],"rather":[49],"than":[50,199],"calibration":[53],"gap":[54,164],"between":[55,79,165],"stated":[56],"confidence":[57],"accuracy.":[59],"To":[60],"examine":[61],"this":[62],"issue,":[63],"we":[64],"introduce":[65],"GIScholarBench,":[66],"benchmark":[68,84],"built":[69],"from":[70],"10,865":[71],"papers":[72],"published":[73],"25":[75],"core":[76],"GIScience":[77],"journals":[78],"2020":[80],"2025.":[82],"The":[83],"covers":[85],"three":[86],"with":[88],"increasing":[89],"cognitive":[90],"complexity:":[91],"metadata":[92,127],"retrieval,":[93,128,218],"literature":[94,149,223],"linking,":[95,150,224],"direction":[98,183],"generation.":[99],"We":[100],"evaluate":[101],"Claude":[102,151],"Sonnet":[103,152],"4.5,":[104],"Gemini":[105],"3,":[106],"ChatGPT":[108,129],"5.3":[109,130],"through":[110],"their":[111],"native":[112],"web":[113],"interfaces":[114],"under":[115],"real-world":[116],"user-facing":[117],"conditions.":[118],"Results":[119],"show":[120,161,187],"consistent":[121],"across":[123],"all":[124,136,159],"tasks.":[125],"In":[126,148,181],"achieves":[131],"highest":[133],"accuracy,":[134],"still":[138],"generate":[139],"definitive":[140],"titles":[141],"DOIs":[143],"predictions":[145],"wrong.":[147],"4.5":[153],"recovers":[154],"most":[156],"references,":[157],"clear":[163],"top-ranked":[166],"retrieval":[167,179],"longer":[169],"citation":[170,220],"lists,":[171],"suggesting":[172],"that":[173,206],"references":[174],"extended":[176],"beyond":[177],"reliable":[178],"capacity.":[180],"generation,":[184],"AI-generated":[185],"directions":[186],"lower":[188,196],"topic":[189],"coverage,":[190],"higher":[191],"novel":[192],"miss":[193],"rates,":[194],"semantic":[197],"diversity":[198],"real":[200],"future-citing":[201],"papers.":[202],"These":[203],"findings":[204],"suggest":[205],"LLM":[207],"task-invariant":[210],"takes":[212],"different":[213],"forms:":[214],"overgeneration":[216],"unreliable":[219],"expansion":[221],"output":[228],"completeness":[229],"during":[230],"ideation.":[232]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-06-10T00:00:00"}