{"id":"https://openalex.org/W4389523875","doi":"https://doi.org/10.18653/v1/2023.findings-emnlp.996","title":"Enhancing Text-to-SQL Capabilities of Large Language Models: A Study on Prompt Design Strategies","display_name":"Enhancing Text-to-SQL Capabilities of Large Language Models: A Study on Prompt Design Strategies","publication_year":2023,"publication_date":"2023-01-01","ids":{"openalex":"https://openalex.org/W4389523875","doi":"https://doi.org/10.18653/v1/2023.findings-emnlp.996"},"language":"en","primary_location":{"id":"doi:10.18653/v1/2023.findings-emnlp.996","doi":"https://doi.org/10.18653/v1/2023.findings-emnlp.996","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2023.findings-emnlp.996","pdf_url":"https://aclanthology.org/2023.findings-emnlp.996.pdf","source":{"id":"https://openalex.org/S7407096085","display_name":"Findings of the Association for Computational Linguistics: EMNLP (Findings of EMNLP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: EMNLP 2023","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","is_traditional":true,"indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://aclanthology.org/2023.findings-emnlp.996.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5026461299","display_name":"Linyong Nan","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I32971472","display_name":"Yale University","ror":"https://ror.org/03v76x132","country_code":"US","type":"education","lineage":["https://openalex.org/I32971472"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Linyong Nan","raw_affiliation_strings":["Yale University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Yale University","institution_ids":["https://openalex.org/I32971472"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5047416722","display_name":"Yilun Zhao","orcid":"https://orcid.org/0000-0002-6812-5120","observed_orcids":["https://orcid.org/0000-0002-6812-5120"]},"institutions":[{"id":"https://openalex.org/I32971472","display_name":"Yale University","ror":"https://ror.org/03v76x132","country_code":"US","type":"education","lineage":["https://openalex.org/I32971472"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yilun Zhao","raw_affiliation_strings":["Yale University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Yale University","institution_ids":["https://openalex.org/I32971472"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5007156413","display_name":"Weijin Zou","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I32971472","display_name":"Yale University","ror":"https://ror.org/03v76x132","country_code":"US","type":"education","lineage":["https://openalex.org/I32971472"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Weijin Zou","raw_affiliation_strings":["Yale University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Yale University","institution_ids":["https://openalex.org/I32971472"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5112921371","display_name":"Narutatsu Ri","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I78577930","display_name":"Columbia University","ror":"https://ror.org/00hj8s172","country_code":"US","type":"education","lineage":["https://openalex.org/I78577930"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Narutatsu Ri","raw_affiliation_strings":["Columbia University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Columbia University","institution_ids":["https://openalex.org/I78577930"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5008425126","display_name":"Jaesung Tae","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I32971472","display_name":"Yale University","ror":"https://ror.org/03v76x132","country_code":"US","type":"education","lineage":["https://openalex.org/I32971472"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jaesung Tae","raw_affiliation_strings":["Yale University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Yale University","institution_ids":["https://openalex.org/I32971472"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5114100725","display_name":"Ellen Zhang","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I32971472","display_name":"Yale University","ror":"https://ror.org/03v76x132","country_code":"US","type":"education","lineage":["https://openalex.org/I32971472"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ellen Zhang","raw_affiliation_strings":["Yale University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Yale University","institution_ids":["https://openalex.org/I32971472"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5064858748","display_name":"Arman Cohan","orcid":"https://orcid.org/0000-0002-8954-2724","observed_orcids":["https://orcid.org/0000-0002-8954-2724"]},"institutions":[{"id":"https://openalex.org/I32971472","display_name":"Yale University","ror":"https://ror.org/03v76x132","country_code":"US","type":"education","lineage":["https://openalex.org/I32971472"]},{"id":"https://openalex.org/I4210156221","display_name":"Allen Institute for Artificial Intelligence","ror":"https://ror.org/05w520734","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210156221"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Arman Cohan","raw_affiliation_strings":["Allen Institute for AI","Yale University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Allen Institute for AI","institution_ids":["https://openalex.org/I4210156221"]},{"raw_affiliation_string":"Yale University","institution_ids":["https://openalex.org/I32971472"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5081787254","display_name":"Dragomir Radev","orcid":"https://orcid.org/0000-0001-7830-6489","observed_orcids":["https://orcid.org/0000-0001-7830-6489"]},"institutions":[{"id":"https://openalex.org/I32971472","display_name":"Yale University","ror":"https://ror.org/03v76x132","country_code":"US","type":"education","lineage":["https://openalex.org/I32971472"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Dragomir Radev","raw_affiliation_strings":["Yale University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Yale University","institution_ids":["https://openalex.org/I32971472"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":11.2681,"has_fulltext":true,"cited_by_count":102,"citation_normalized_percentile":{"value":0.98995703,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":97,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"14935","last_page":"14956"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.7060999870300293,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.7060999870300293,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.14980000257492065,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10317","display_name":"Advanced Database Systems and Queries","score":0.02539999969303608,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/text-to-sql","display_name":"text-to-SQL","score":0.9319999814033508},{"id":"https://openalex.org/keywords/prompt-engineering","display_name":"prompt engineering","score":0.7459999918937683},{"id":"https://openalex.org/keywords/in-context-learning","display_name":"in-context learning","score":0.9919999837875366},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.8769999742507935},{"id":"https://openalex.org/keywords/demonstration-selection","display_name":"demonstration selection","score":0.9729999899864197},{"id":"https://openalex.org/keywords/instructional-format","display_name":"instructional format","score":0.5720000267028809},{"id":"https://openalex.org/keywords/spider-dataset","display_name":"Spider dataset","score":0.6079999804496765},{"id":"https://openalex.org/keywords/execution-accuracy","display_name":"execution accuracy","score":0.6069999933242798}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8519445061683655},{"id":"https://openalex.org/C510870499","wikidata":"https://www.wikidata.org/wiki/Q47607","display_name":"SQL","level":2,"score":0.7915689945220947},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.6273509860038757},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.612139880657196},{"id":"https://openalex.org/C81917197","wikidata":"https://www.wikidata.org/wiki/Q628760","display_name":"Selection (genetic algorithm)","level":2,"score":0.5066186189651489},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.43135330080986023},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.40588629245758057},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.24090644717216492},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.18653/v1/2023.findings-emnlp.996","doi":"https://doi.org/10.18653/v1/2023.findings-emnlp.996","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2023.findings-emnlp.996","pdf_url":"https://aclanthology.org/2023.findings-emnlp.996.pdf","source":{"id":"https://openalex.org/S7407096085","display_name":"Findings of the Association for Computational Linguistics: EMNLP (Findings of EMNLP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: EMNLP 2023","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2305.12586","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2305.12586","pdf_url":"https://arxiv.org/pdf/2305.12586","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2305.12586","doi":"https://doi.org/10.48550/arxiv.2305.12586","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2305.12586","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.18653/v1/2023.findings-emnlp.996","doi":"https://doi.org/10.18653/v1/2023.findings-emnlp.996","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2023.findings-emnlp.996","pdf_url":"https://aclanthology.org/2023.findings-emnlp.996.pdf","source":{"id":"https://openalex.org/S7407096085","display_name":"Findings of the Association for Computational Linguistics: EMNLP (Findings of EMNLP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: EMNLP 2023","raw_type":"proceedings-article"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4389523875.pdf","grobid_xml":"https://content.openalex.org/works/W4389523875.grobid-xml"},"referenced_works_count":38,"referenced_works":["https://openalex.org/W2148721079","https://openalex.org/W2252136820","https://openalex.org/W2546950329","https://openalex.org/W2751448157","https://openalex.org/W2884709258","https://openalex.org/W2890431379","https://openalex.org/W2890867094","https://openalex.org/W2970442801","https://openalex.org/W2970641574","https://openalex.org/W2981852735","https://openalex.org/W2985377636","https://openalex.org/W3030163527","https://openalex.org/W3035231859","https://openalex.org/W3093819145","https://openalex.org/W3098605233","https://openalex.org/W3122241445","https://openalex.org/W3156470785","https://openalex.org/W3175818566","https://openalex.org/W3177813494","https://openalex.org/W3198685994","https://openalex.org/W3199077625","https://openalex.org/W4221161695","https://openalex.org/W4226053975","https://openalex.org/W4297633153","https://openalex.org/W4302011807","https://openalex.org/W4303649020","https://openalex.org/W4308244910","https://openalex.org/W4309953112","https://openalex.org/W4321277276","https://openalex.org/W4322718191","https://openalex.org/W4361019453","https://openalex.org/W4382202531","https://openalex.org/W4385572953","https://openalex.org/W4385573504","https://openalex.org/W4389520103","https://openalex.org/W4389520255","https://openalex.org/W7133224126","https://openalex.org/W7133231369"],"related_works":["https://openalex.org/W4377864495","https://openalex.org/W4404467346","https://openalex.org/W4396571402","https://openalex.org/W4389519829","https://openalex.org/W4379258895","https://openalex.org/W4400611569","https://openalex.org/W4396913744","https://openalex.org/W7165162669","https://openalex.org/W4409601911","https://openalex.org/W4388639106"],"abstract_inverted_index":{"In-context":[0],"learning":[1],"(ICL)":[2],"has":[3,27],"emerged":[4],"as":[5],"a":[6,31,68],"new":[7],"approach":[8,87,156],"to":[9,20,41,45,98,114,160,173],"various":[10,60],"natural":[11],"language":[12,17],"processing":[13],"tasks,":[14],"utilizing":[15],"large":[16],"models":[18],"(LLMs)":[19],"make":[21],"predictions":[22],"based":[23],"on":[24,146],"context":[25],"that":[26,49,104,119],"been":[28],"supplemented":[29],"with":[30],"few":[32],"examples":[33],"or":[34],"task-specific":[35],"instructions.In":[36],"this":[37,43],"paper,":[38],"we":[39,102,117,165],"aim":[40],"extend":[42],"method":[44],"question":[46],"answering":[47],"tasks":[48],"utilize":[50],"structured":[51],"knowledge":[52,124],"sources,":[53],"and":[54,76,101,108,138,164],"improve":[55],"Text-to-SQL":[56,85,162],"systems":[57],"by":[58,133,143],"exploring":[59],"prompt":[61],"design":[62],"strategies":[63],"for":[64,80],"employing":[65],"LLMs.We":[66],"conduct":[67],"systematic":[69],"investigation":[70],"into":[71],"different":[72],"demonstration":[73,111],"selection":[74,112],"methods":[75],"optimal":[77],"instruction":[78],"formats":[79],"prompting":[81],"LLMs":[82,120,159],"in":[83,110,157],"the":[84,90,130,139,147,152,161,170,174],"task.Our":[86],"involves":[88],"leveraging":[89],"syntactic":[91],"structure":[92],"of":[93,154,169,176],"an":[94,167],"example's":[95],"SQL":[96],"query":[97],"retrieve":[99],"demonstrations,":[100],"demonstrate":[103],"pursuing":[105],"both":[106],"diversity":[107],"similarity":[109],"leads":[113],"enhanced":[115],"performance.Furthermore,":[116],"show":[118],"benefit":[121],"from":[122],"database-related":[123],"augmentations.Our":[125],"most":[126],"effective":[127],"strategy":[128],"outperforms":[129],"state-of-the-art":[131],"system":[132,142],"2.5":[134],"points":[135,145],"(Execution":[136],"Accuracy)":[137],"best":[140],"fine-tuned":[141],"5.1":[144],"Spider":[148],"dataset.These":[149],"results":[150],"highlight":[151],"effectiveness":[153],"our":[155,177],"adapting":[158],"task,":[163],"present":[166],"analysis":[168],"factors":[171],"contributing":[172],"success":[175],"strategy.":[178]},"counts_by_year":[{"year":2026,"cited_by_count":26},{"year":2025,"cited_by_count":43},{"year":2024,"cited_by_count":28},{"year":2023,"cited_by_count":5}],"updated_date":"2026-11-10T11:39:43.021295","created_date":"2023-12-11T00:00:00"}