{"id":"https://openalex.org/W4392453936","doi":"https://doi.org/10.14778/3636218.3636225","title":"ScienceBenchmark: A Complex Real-World Benchmark for Evaluating Natural Language to SQL Systems","display_name":"ScienceBenchmark: A Complex Real-World Benchmark for Evaluating Natural Language to SQL Systems","publication_year":2023,"publication_date":"2023-12-01","ids":{"openalex":"https://openalex.org/W4392453936","doi":"https://doi.org/10.14778/3636218.3636225"},"language":"en","primary_location":{"id":"doi:10.14778/3636218.3636225","is_oa":false,"landing_page_url":"https://doi.org/10.14778/3636218.3636225","pdf_url":null,"source":{"id":"https://openalex.org/S4210226185","display_name":"Proceedings of the VLDB Endowment","issn_l":"2150-8097","issn":["2150-8097"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the VLDB Endowment","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://hdl.handle.net/11475/30173","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100388367","display_name":"Yi Zhang","orcid":"https://orcid.org/0009-0007-7269-8187"},"institutions":[{"id":"https://openalex.org/I858936495","display_name":"ZHAW Zurich University of Applied Sciences","ror":"https://ror.org/05pmsvm27","country_code":"CH","type":"education","lineage":["https://openalex.org/I858936495"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Yi Zhang","raw_affiliation_strings":["Zurich University of Applied Sciences, Switzerland"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Zurich University of Applied Sciences, Switzerland","institution_ids":["https://openalex.org/I858936495"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5018584668","display_name":"Jan Deriu","orcid":"https://orcid.org/0000-0002-8405-1344"},"institutions":[{"id":"https://openalex.org/I858936495","display_name":"ZHAW Zurich University of Applied Sciences","ror":"https://ror.org/05pmsvm27","country_code":"CH","type":"education","lineage":["https://openalex.org/I858936495"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Jan Deriu","raw_affiliation_strings":["Zurich University of Applied Sciences, Switzerland"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Zurich University of Applied Sciences, Switzerland","institution_ids":["https://openalex.org/I858936495"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5027131827","display_name":"George Katsogiannis-Meimarakis","orcid":"https://orcid.org/0000-0001-8285-5827"},"institutions":[{"id":"https://openalex.org/I4210156054","display_name":"Athena Research and Innovation Center In Information Communication & Knowledge Technologies","ror":"https://ror.org/0576by029","country_code":"GR","type":"facility","lineage":["https://openalex.org/I4210156054"]}],"countries":["GR"],"is_corresponding":false,"raw_author_name":"George Katsogiannis-Meimarakis","raw_affiliation_strings":["Athena Research Center, Greece"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Athena Research Center, Greece","institution_ids":["https://openalex.org/I4210156054"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5091819244","display_name":"Catherine Kosten","orcid":null},"institutions":[{"id":"https://openalex.org/I858936495","display_name":"ZHAW Zurich University of Applied Sciences","ror":"https://ror.org/05pmsvm27","country_code":"CH","type":"education","lineage":["https://openalex.org/I858936495"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Catherine Kosten","raw_affiliation_strings":["Zurich University of Applied Sciences, Switzerland"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Zurich University of Applied Sciences, Switzerland","institution_ids":["https://openalex.org/I858936495"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5012623045","display_name":"Georgia Koutrika","orcid":"https://orcid.org/0000-0002-7377-0116"},"institutions":[{"id":"https://openalex.org/I4210156054","display_name":"Athena Research and Innovation Center In Information Communication & Knowledge Technologies","ror":"https://ror.org/0576by029","country_code":"GR","type":"facility","lineage":["https://openalex.org/I4210156054"]}],"countries":["GR"],"is_corresponding":false,"raw_author_name":"Georgia Koutrika","raw_affiliation_strings":["Athena Research Center, Greece"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Athena Research Center, Greece","institution_ids":["https://openalex.org/I4210156054"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5074825049","display_name":"Kurt Stockinger","orcid":"https://orcid.org/0000-0003-4034-4812"},"institutions":[{"id":"https://openalex.org/I858936495","display_name":"ZHAW Zurich University of Applied Sciences","ror":"https://ror.org/05pmsvm27","country_code":"CH","type":"education","lineage":["https://openalex.org/I858936495"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Kurt Stockinger","raw_affiliation_strings":["Zurich University of Applied Sciences, Switzerland"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Zurich University of Applied Sciences, Switzerland","institution_ids":["https://openalex.org/I858936495"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":6,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":4.7317,"has_fulltext":false,"cited_by_count":29,"citation_normalized_percentile":{"value":0.96035485,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":97,"max":100},"biblio":{"volume":"17","issue":"4","first_page":"685","last_page":"698"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10215","display_name":"Semantic Web and Ontologies","score":0.9977999925613403,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6645532846450806},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.5818911194801331},{"id":"https://openalex.org/keywords/sql","display_name":"SQL","score":0.5463956594467163},{"id":"https://openalex.org/keywords/data-definition-language","display_name":"Data definition language","score":0.4937227666378021},{"id":"https://openalex.org/keywords/natural","display_name":"Natural (archaeology)","score":0.4495373070240021},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.4351779818534851},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.34344130754470825},{"id":"https://openalex.org/keywords/history","display_name":"History","score":0.15608617663383484},{"id":"https://openalex.org/keywords/geography","display_name":"Geography","score":0.12289968132972717},{"id":"https://openalex.org/keywords/cartography","display_name":"Cartography","score":0.08960217237472534},{"id":"https://openalex.org/keywords/archaeology","display_name":"Archaeology","score":0.054648011922836304}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6645532846450806},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.5818911194801331},{"id":"https://openalex.org/C510870499","wikidata":"https://www.wikidata.org/wiki/Q47607","display_name":"SQL","level":2,"score":0.5463956594467163},{"id":"https://openalex.org/C55596503","wikidata":"https://www.wikidata.org/wiki/Q1431648","display_name":"Data definition language","level":3,"score":0.4937227666378021},{"id":"https://openalex.org/C2776608160","wikidata":"https://www.wikidata.org/wiki/Q4785462","display_name":"Natural (archaeology)","level":2,"score":0.4495373070240021},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.4351779818534851},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.34344130754470825},{"id":"https://openalex.org/C95457728","wikidata":"https://www.wikidata.org/wiki/Q309","display_name":"History","level":0,"score":0.15608617663383484},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.12289968132972717},{"id":"https://openalex.org/C58640448","wikidata":"https://www.wikidata.org/wiki/Q42515","display_name":"Cartography","level":1,"score":0.08960217237472534},{"id":"https://openalex.org/C166957645","wikidata":"https://www.wikidata.org/wiki/Q23498","display_name":"Archaeology","level":1,"score":0.054648011922836304}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.14778/3636218.3636225","is_oa":false,"landing_page_url":"https://doi.org/10.14778/3636218.3636225","pdf_url":null,"source":{"id":"https://openalex.org/S4210226185","display_name":"Proceedings of the VLDB Endowment","issn_l":"2150-8097","issn":["2150-8097"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the VLDB Endowment","raw_type":"journal-article"},{"id":"pmh:oai:digitalcollection.zhaw.ch:11475/30173","is_oa":true,"landing_page_url":"https://hdl.handle.net/11475/30173","pdf_url":null,"source":{"id":"https://openalex.org/S4306401810","display_name":"Z\u00fcrcher Hochschule f\u00fcr Angewandte Wissenschaften digital collection (Zurich University of Applied Sciences)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I200744771","host_organization_name":"ZHAW Zurich University of Applied Sciences","host_organization_lineage":["https://openalex.org/I200744771"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Text"}],"best_oa_location":{"id":"pmh:oai:digitalcollection.zhaw.ch:11475/30173","is_oa":true,"landing_page_url":"https://hdl.handle.net/11475/30173","pdf_url":null,"source":{"id":"https://openalex.org/S4306401810","display_name":"Z\u00fcrcher Hochschule f\u00fcr Angewandte Wissenschaften digital collection (Zurich University of Applied Sciences)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I200744771","host_organization_name":"ZHAW Zurich University of Applied Sciences","host_organization_lineage":["https://openalex.org/I200744771"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":39,"referenced_works":["https://openalex.org/W1970709149","https://openalex.org/W2032374895","https://openalex.org/W2084961399","https://openalex.org/W2101105183","https://openalex.org/W2145618437","https://openalex.org/W2153710242","https://openalex.org/W2154268919","https://openalex.org/W2250829141","https://openalex.org/W2269738476","https://openalex.org/W2536574992","https://openalex.org/W2740656274","https://openalex.org/W2762513422","https://openalex.org/W2798663534","https://openalex.org/W2886983157","https://openalex.org/W2890431379","https://openalex.org/W2945102109","https://openalex.org/W2950067395","https://openalex.org/W2963532001","https://openalex.org/W2964271186","https://openalex.org/W2970641574","https://openalex.org/W3022196370","https://openalex.org/W3032766766","https://openalex.org/W3034750657","https://openalex.org/W3034835156","https://openalex.org/W3086973390","https://openalex.org/W3094344961","https://openalex.org/W3104055253","https://openalex.org/W3115880855","https://openalex.org/W3134503905","https://openalex.org/W3170978252","https://openalex.org/W3172094450","https://openalex.org/W3175818566","https://openalex.org/W3185858112","https://openalex.org/W3200079259","https://openalex.org/W4210716649","https://openalex.org/W4224239544","https://openalex.org/W4288601872","https://openalex.org/W4317716303","https://openalex.org/W4382202531"],"related_works":["https://openalex.org/W1592149428","https://openalex.org/W650213220","https://openalex.org/W1980548249","https://openalex.org/W2183714398","https://openalex.org/W3094396153","https://openalex.org/W2042813897","https://openalex.org/W2340883001","https://openalex.org/W4382050951","https://openalex.org/W2079321164","https://openalex.org/W1554917839"],"abstract_inverted_index":{"Natural":[0],"Language":[1],"to":[2,16,21,63,74],"SQL":[3,17,103],"systems":[4,144,162],"(NL-to-SQL)":[5],"have":[6,61],"recently":[7],"shown":[8],"improved":[9],"accuracy":[10],"(exceeding":[11],"80%)":[12],"for":[13,93,111,163],"natural":[14],"language":[15,26],"query":[18],"translation":[19],"due":[20],"the":[22,29,32,69,120,141,156,184],"emergence":[23],"of":[24,31,71,77,123,171],"transformer-based":[25],"models,":[27],"and":[28,45,105,197],"popularity":[30],"Spider":[33,36,146],"benchmark.":[34,154],"However,":[35],"mainly":[37],"contains":[38],"simple":[39],"databases":[40,57],"with":[41,58,126,167,176,189],"few":[42],"tables,":[43],"columns,":[44],"entries,":[46],"which":[47],"do":[48],"not":[49],"reflect":[50],"a":[51,88,148,168],"realistic":[52],"setting.":[53],"Moreover,":[54],"complex":[55,90,165,190],"real-world":[56,191],"domain-specific":[59,97],"content":[60],"little":[62],"no":[64],"training":[65,173,196],"data":[66,125,128,174,199],"available":[67],"in":[68],"form":[70],"NL/SQL-pairs":[72,110],"leading":[73],"poor":[75],"performance":[76,151],"existing":[78],"NL-to-SQL":[79,91,161,186],"systems.":[80],"In":[81],"this":[82,100],"paper,":[83],"we":[84,118],"introduce":[85],"ScienceBenchmark":[86,182],",":[87],"new":[89,101],"benchmark":[92,136,187],"three":[94],"real-world,":[95],"highly":[96,138,164],"databases.":[98],"For":[99],"benchmark,":[102],"experts":[104,107],"domain":[106,203],"created":[108],"high-quality":[109],"each":[112],"domain.":[113],"To":[114,179],"garner":[115],"more":[116],"data,":[117],"extended":[119],"small":[121,169],"amount":[122,170],"human-generated":[124],"synthetic":[127,177],"generated":[129],"using":[130],"GPT-3.":[131],"We":[132],"show":[133],"that":[134],"our":[135,153,180],"is":[137,158,183],"challenging,":[139],"as":[140],"top":[142],"performing":[143],"on":[145,152],"achieve":[147],"very":[149],"low":[150],"Thus,":[155],"challenge":[157],"many-fold:":[159],"creating":[160],"domains":[166],"hand-made":[172],"augmented":[175],"data.":[178],"knowledge,":[181],"first":[185],"designed":[188],"scientific":[192],"databases,":[193],"containing":[194],"challenging":[195],"test":[198],"carefully":[200],"validated":[201],"by":[202],"experts.":[204]},"counts_by_year":[{"year":2026,"cited_by_count":7},{"year":2025,"cited_by_count":18},{"year":2024,"cited_by_count":4}],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-10-10T00:00:00"}