{"id":"https://openalex.org/W4414976110","doi":"https://doi.org/10.48550/arxiv.2510.05318","title":"BIRD-INTERACT: Re-imagining Text-to-SQL Evaluation for Large Language Models via Lens of Dynamic Interactions","display_name":"BIRD-INTERACT: Re-imagining Text-to-SQL Evaluation for Large Language Models via Lens of Dynamic Interactions","publication_year":2025,"publication_date":"2025-10-06","ids":{"openalex":"https://openalex.org/W4414976110","doi":"https://doi.org/10.48550/arxiv.2510.05318"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2510.05318","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2510.05318","pdf_url":"https://arxiv.org/pdf/2510.05318","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2510.05318","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5071652041","display_name":"Nan Huo","orcid":"https://orcid.org/0000-0002-9305-3309","observed_orcids":["https://orcid.org/0000-0002-9305-3309"]},"institutions":[{"id":"https://openalex.org/I889458895","display_name":"University of Hong Kong","ror":"https://ror.org/02zhqgq86","country_code":"HK","type":"education","lineage":["https://openalex.org/I889458895"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Huo, Nan","raw_affiliation_strings":["\u03b1 The University of Hong Kong \u03b2 Google Cloud \u03b3 The BIRD Team"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"\u03b1 The University of Hong Kong \u03b2 Google Cloud \u03b3 The BIRD Team","institution_ids":["https://openalex.org/I889458895"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Xu, Xiaohan","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I889458895","display_name":"University of Hong Kong","ror":"https://ror.org/02zhqgq86","country_code":"HK","type":"education","lineage":["https://openalex.org/I889458895"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Xu, Xiaohan","raw_affiliation_strings":["\u03b1 The University of Hong Kong \u03b2 Google Cloud \u03b3 The BIRD Team"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"\u03b1 The University of Hong Kong \u03b2 Google Cloud \u03b3 The BIRD Team","institution_ids":["https://openalex.org/I889458895"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Li, Jinyang","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I889458895","display_name":"University of Hong Kong","ror":"https://ror.org/02zhqgq86","country_code":"HK","type":"education","lineage":["https://openalex.org/I889458895"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Li, Jinyang","raw_affiliation_strings":["\u03b1 The University of Hong Kong \u03b2 Google Cloud \u03b3 The BIRD Team"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"\u03b1 The University of Hong Kong \u03b2 Google Cloud \u03b3 The BIRD Team","institution_ids":["https://openalex.org/I889458895"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5119788458","display_name":"Per Jacobsson","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I889458895","display_name":"University of Hong Kong","ror":"https://ror.org/02zhqgq86","country_code":"HK","type":"education","lineage":["https://openalex.org/I889458895"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Jacobsson, Per","raw_affiliation_strings":["\u03b1 The University of Hong Kong \u03b2 Google Cloud \u03b3 The BIRD Team"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"\u03b1 The University of Hong Kong \u03b2 Google Cloud \u03b3 The BIRD Team","institution_ids":["https://openalex.org/I889458895"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Lin, Shipei","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I889458895","display_name":"University of Hong Kong","ror":"https://ror.org/02zhqgq86","country_code":"HK","type":"education","lineage":["https://openalex.org/I889458895"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Lin, Shipei","raw_affiliation_strings":["\u03b1 The University of Hong Kong \u03b2 Google Cloud \u03b3 The BIRD Team"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"\u03b1 The University of Hong Kong \u03b2 Google Cloud \u03b3 The BIRD Team","institution_ids":["https://openalex.org/I889458895"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Qin, Bowen","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I889458895","display_name":"University of Hong Kong","ror":"https://ror.org/02zhqgq86","country_code":"HK","type":"education","lineage":["https://openalex.org/I889458895"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Qin, Bowen","raw_affiliation_strings":["\u03b1 The University of Hong Kong \u03b2 Google Cloud \u03b3 The BIRD Team"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"\u03b1 The University of Hong Kong \u03b2 Google Cloud \u03b3 The BIRD Team","institution_ids":["https://openalex.org/I889458895"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5080628250","display_name":"Binyuan Hui","orcid":"https://orcid.org/0000-0002-2160-7595","observed_orcids":["https://orcid.org/0000-0002-2160-7595"]},"institutions":[{"id":"https://openalex.org/I889458895","display_name":"University of Hong Kong","ror":"https://ror.org/02zhqgq86","country_code":"HK","type":"education","lineage":["https://openalex.org/I889458895"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Hui, Binyuan","raw_affiliation_strings":["\u03b1 The University of Hong Kong \u03b2 Google Cloud \u03b3 The BIRD Team"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"\u03b1 The University of Hong Kong \u03b2 Google Cloud \u03b3 The BIRD Team","institution_ids":["https://openalex.org/I889458895"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100371535","display_name":"Xiaolong Li","orcid":"https://orcid.org/0000-0001-7493-2650","observed_orcids":["https://orcid.org/0000-0001-7493-2650","https://orcid.org/0009-0009-6586-9077"]},"institutions":[{"id":"https://openalex.org/I889458895","display_name":"University of Hong Kong","ror":"https://ror.org/02zhqgq86","country_code":"HK","type":"education","lineage":["https://openalex.org/I889458895"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Li, Xiaolong","raw_affiliation_strings":["\u03b1 The University of Hong Kong \u03b2 Google Cloud \u03b3 The BIRD Team"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"\u03b1 The University of Hong Kong \u03b2 Google Cloud \u03b3 The BIRD Team","institution_ids":["https://openalex.org/I889458895"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5094123544","display_name":"Ge Qu","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I889458895","display_name":"University of Hong Kong","ror":"https://ror.org/02zhqgq86","country_code":"HK","type":"education","lineage":["https://openalex.org/I889458895"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Qu, Ge","raw_affiliation_strings":["\u03b1 The University of Hong Kong \u03b2 Google Cloud \u03b3 The BIRD Team"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"\u03b1 The University of Hong Kong \u03b2 Google Cloud \u03b3 The BIRD Team","institution_ids":["https://openalex.org/I889458895"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5022397026","display_name":"Shuzheng Si","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Si, Shuzheng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Han, Linheng","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I889458895","display_name":"University of Hong Kong","ror":"https://ror.org/02zhqgq86","country_code":"HK","type":"education","lineage":["https://openalex.org/I889458895"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Han, Linheng","raw_affiliation_strings":["\u03b1 The University of Hong Kong \u03b2 Google Cloud \u03b3 The BIRD Team"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"\u03b1 The University of Hong Kong \u03b2 Google Cloud \u03b3 The BIRD Team","institution_ids":["https://openalex.org/I889458895"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5119913710","display_name":"Edward Alexander","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I889458895","display_name":"University of Hong Kong","ror":"https://ror.org/02zhqgq86","country_code":"HK","type":"education","lineage":["https://openalex.org/I889458895"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Alexander, Edward","raw_affiliation_strings":["\u03b1 The University of Hong Kong \u03b2 Google Cloud \u03b3 The BIRD Team"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"\u03b1 The University of Hong Kong \u03b2 Google Cloud \u03b3 The BIRD Team","institution_ids":["https://openalex.org/I889458895"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5048495303","display_name":"Xintong Zhu","orcid":"https://orcid.org/0000-0001-9037-9111","observed_orcids":["https://orcid.org/0000-0001-9037-9111"]},"institutions":[{"id":"https://openalex.org/I889458895","display_name":"University of Hong Kong","ror":"https://ror.org/02zhqgq86","country_code":"HK","type":"education","lineage":["https://openalex.org/I889458895"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Zhu, Xintong","raw_affiliation_strings":["\u03b1 The University of Hong Kong \u03b2 Google Cloud \u03b3 The BIRD Team"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"\u03b1 The University of Hong Kong \u03b2 Google Cloud \u03b3 The BIRD Team","institution_ids":["https://openalex.org/I889458895"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Qin, Rui","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I889458895","display_name":"University of Hong Kong","ror":"https://ror.org/02zhqgq86","country_code":"HK","type":"education","lineage":["https://openalex.org/I889458895"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Qin, Rui","raw_affiliation_strings":["\u03b1 The University of Hong Kong \u03b2 Google Cloud \u03b3 The BIRD Team"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"\u03b1 The University of Hong Kong \u03b2 Google Cloud \u03b3 The BIRD Team","institution_ids":["https://openalex.org/I889458895"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Yu, Ruihan","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I889458895","display_name":"University of Hong Kong","ror":"https://ror.org/02zhqgq86","country_code":"HK","type":"education","lineage":["https://openalex.org/I889458895"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Yu, Ruihan","raw_affiliation_strings":["\u03b1 The University of Hong Kong \u03b2 Google Cloud \u03b3 The BIRD Team"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"\u03b1 The University of Hong Kong \u03b2 Google Cloud \u03b3 The BIRD Team","institution_ids":["https://openalex.org/I889458895"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Jin, Yiyao","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I889458895","display_name":"University of Hong Kong","ror":"https://ror.org/02zhqgq86","country_code":"HK","type":"education","lineage":["https://openalex.org/I889458895"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Jin, Yiyao","raw_affiliation_strings":["\u03b1 The University of Hong Kong \u03b2 Google Cloud \u03b3 The BIRD Team"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"\u03b1 The University of Hong Kong \u03b2 Google Cloud \u03b3 The BIRD Team","institution_ids":["https://openalex.org/I889458895"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Zhou, Feige","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I889458895","display_name":"University of Hong Kong","ror":"https://ror.org/02zhqgq86","country_code":"HK","type":"education","lineage":["https://openalex.org/I889458895"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Zhou, Feige","raw_affiliation_strings":["\u03b1 The University of Hong Kong \u03b2 Google Cloud \u03b3 The BIRD Team"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"\u03b1 The University of Hong Kong \u03b2 Google Cloud \u03b3 The BIRD Team","institution_ids":["https://openalex.org/I889458895"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Zhong, Weihao","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I889458895","display_name":"University of Hong Kong","ror":"https://ror.org/02zhqgq86","country_code":"HK","type":"education","lineage":["https://openalex.org/I889458895"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Zhong, Weihao","raw_affiliation_strings":["\u03b1 The University of Hong Kong \u03b2 Google Cloud \u03b3 The BIRD Team"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"\u03b1 The University of Hong Kong \u03b2 Google Cloud \u03b3 The BIRD Team","institution_ids":["https://openalex.org/I889458895"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100416565","display_name":"Yun Chen","orcid":"https://orcid.org/0000-0002-9934-9979","observed_orcids":["https://orcid.org/0000-0002-9934-9979"]},"institutions":[{"id":"https://openalex.org/I889458895","display_name":"University of Hong Kong","ror":"https://ror.org/02zhqgq86","country_code":"HK","type":"education","lineage":["https://openalex.org/I889458895"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Chen, Yun","raw_affiliation_strings":["\u03b1 The University of Hong Kong \u03b2 Google Cloud \u03b3 The BIRD Team"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"\u03b1 The University of Hong Kong \u03b2 Google Cloud \u03b3 The BIRD Team","institution_ids":["https://openalex.org/I889458895"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Liu, Hongyu","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I889458895","display_name":"University of Hong Kong","ror":"https://ror.org/02zhqgq86","country_code":"HK","type":"education","lineage":["https://openalex.org/I889458895"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Liu, Hongyu","raw_affiliation_strings":["\u03b1 The University of Hong Kong \u03b2 Google Cloud \u03b3 The BIRD Team"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"\u03b1 The University of Hong Kong \u03b2 Google Cloud \u03b3 The BIRD Team","institution_ids":["https://openalex.org/I889458895"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5055857919","display_name":"Chenhao Ma","orcid":"https://orcid.org/0000-0002-3243-8512","observed_orcids":["https://orcid.org/0000-0002-3243-8512"]},"institutions":[{"id":"https://openalex.org/I889458895","display_name":"University of Hong Kong","ror":"https://ror.org/02zhqgq86","country_code":"HK","type":"education","lineage":["https://openalex.org/I889458895"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Ma, Chenhao","raw_affiliation_strings":["\u03b1 The University of Hong Kong \u03b2 Google Cloud \u03b3 The BIRD Team"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"\u03b1 The University of Hong Kong \u03b2 Google Cloud \u03b3 The BIRD Team","institution_ids":["https://openalex.org/I889458895"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5070638387","display_name":"Fatma \u00d6zcan","orcid":"https://orcid.org/0000-0002-4418-4724","observed_orcids":["https://orcid.org/0000-0002-4418-4724"]},"institutions":[{"id":"https://openalex.org/I889458895","display_name":"University of Hong Kong","ror":"https://ror.org/02zhqgq86","country_code":"HK","type":"education","lineage":["https://openalex.org/I889458895"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Ozcan, Fatma","raw_affiliation_strings":["\u03b1 The University of Hong Kong \u03b2 Google Cloud \u03b3 The BIRD Team"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"\u03b1 The University of Hong Kong \u03b2 Google Cloud \u03b3 The BIRD Team","institution_ids":["https://openalex.org/I889458895"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5059167302","display_name":"Yannis Papakonstantinou","orcid":"https://orcid.org/0009-0007-6360-9496","observed_orcids":["https://orcid.org/0009-0007-6360-9496"]},"institutions":[{"id":"https://openalex.org/I889458895","display_name":"University of Hong Kong","ror":"https://ror.org/02zhqgq86","country_code":"HK","type":"education","lineage":["https://openalex.org/I889458895"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Papakonstantinou, Yannis","raw_affiliation_strings":["\u03b1 The University of Hong Kong \u03b2 Google Cloud \u03b3 The BIRD Team"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"\u03b1 The University of Hong Kong \u03b2 Google Cloud \u03b3 The BIRD Team","institution_ids":["https://openalex.org/I889458895"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5005847882","display_name":"Reynold C. K. Cheng","orcid":"https://orcid.org/0000-0002-9480-9809","observed_orcids":["https://orcid.org/0000-0002-9480-9809"]},"institutions":[{"id":"https://openalex.org/I889458895","display_name":"University of Hong Kong","ror":"https://ror.org/02zhqgq86","country_code":"HK","type":"education","lineage":["https://openalex.org/I889458895"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Cheng, Reynold","raw_affiliation_strings":["\u03b1 The University of Hong Kong \u03b2 Google Cloud \u03b3 The BIRD Team"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"\u03b1 The University of Hong Kong \u03b2 Google Cloud \u03b3 The BIRD Team","institution_ids":["https://openalex.org/I889458895"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":27,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.32710000872612,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.32710000872612,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.1996999979019165,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10317","display_name":"Advanced Database Systems and Queries","score":0.1665000021457672,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/text-to-sql","display_name":"text-to-SQL","score":0.9639999866485596},{"id":"https://openalex.org/keywords/multi-turn-interaction","display_name":"multi-turn interaction","score":0.7739999890327454},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.7630000114440918},{"id":"https://openalex.org/keywords/benchmark","display_name":"benchmark","score":0.5220000147819519},{"id":"https://openalex.org/keywords/user-simulator","display_name":"user simulator","score":0.6380000114440918},{"id":"https://openalex.org/keywords/ambiguous-queries","display_name":"ambiguous queries","score":0.5630000233650208},{"id":"https://openalex.org/keywords/error-recovery","display_name":"error recovery","score":0.8090000152587891},{"id":"https://openalex.org/keywords/crud-operations","display_name":"CRUD operations","score":0.574999988079071}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8479999899864197},{"id":"https://openalex.org/C160145156","wikidata":"https://www.wikidata.org/wiki/Q778586","display_name":"Executable","level":2,"score":0.7246999740600586},{"id":"https://openalex.org/C79581498","wikidata":"https://www.wikidata.org/wiki/Q1367530","display_name":"Suite","level":2,"score":0.5982999801635742},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.5958999991416931},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.5133000016212463},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.512499988079071},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.5006999969482422},{"id":"https://openalex.org/C51929080","wikidata":"https://www.wikidata.org/wiki/Q2425187","display_name":"Codebase","level":3,"score":0.4584999978542328},{"id":"https://openalex.org/C2777200299","wikidata":"https://www.wikidata.org/wiki/Q52943","display_name":"Conversation","level":2,"score":0.4300999939441681},{"id":"https://openalex.org/C93518851","wikidata":"https://www.wikidata.org/wiki/Q180160","display_name":"Metadata","level":2,"score":0.4020000100135803},{"id":"https://openalex.org/C175154964","wikidata":"https://www.wikidata.org/wiki/Q380077","display_name":"Task analysis","level":3,"score":0.3944999873638153},{"id":"https://openalex.org/C168065819","wikidata":"https://www.wikidata.org/wiki/Q845566","display_name":"Debugging","level":2,"score":0.3824999928474426},{"id":"https://openalex.org/C151552104","wikidata":"https://www.wikidata.org/wiki/Q7705809","display_name":"Test suite","level":4,"score":0.37929999828338623},{"id":"https://openalex.org/C89505385","wikidata":"https://www.wikidata.org/wiki/Q47146","display_name":"User interface","level":2,"score":0.36329999566078186},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3531999886035919},{"id":"https://openalex.org/C184337299","wikidata":"https://www.wikidata.org/wiki/Q1437428","display_name":"Semantics (computer science)","level":2,"score":0.33799999952316284},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.33739998936653137},{"id":"https://openalex.org/C170130773","wikidata":"https://www.wikidata.org/wiki/Q216378","display_name":"Usability","level":2,"score":0.3310999870300293},{"id":"https://openalex.org/C194995250","wikidata":"https://www.wikidata.org/wiki/Q531136","display_name":"Affordance","level":2,"score":0.32710000872612},{"id":"https://openalex.org/C77618280","wikidata":"https://www.wikidata.org/wiki/Q1155772","display_name":"Scheme (mathematics)","level":2,"score":0.3174999952316284},{"id":"https://openalex.org/C2780385302","wikidata":"https://www.wikidata.org/wiki/Q367158","display_name":"Protocol (science)","level":3,"score":0.3160000145435333},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.31029999256134033},{"id":"https://openalex.org/C45357846","wikidata":"https://www.wikidata.org/wiki/Q2001982","display_name":"Notation","level":2,"score":0.28690001368522644},{"id":"https://openalex.org/C38764148","wikidata":"https://www.wikidata.org/wiki/Q17098245","display_name":"Interaction information","level":2,"score":0.28189998865127563},{"id":"https://openalex.org/C113843644","wikidata":"https://www.wikidata.org/wiki/Q901882","display_name":"Interface (matter)","level":4,"score":0.2750999927520752},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.27480000257492065},{"id":"https://openalex.org/C75291252","wikidata":"https://www.wikidata.org/wiki/Q1315756","display_name":"TRACE (psycholinguistics)","level":2,"score":0.27250000834465027},{"id":"https://openalex.org/C144986985","wikidata":"https://www.wikidata.org/wiki/Q871236","display_name":"Hierarchical database model","level":2,"score":0.27079999446868896},{"id":"https://openalex.org/C67186912","wikidata":"https://www.wikidata.org/wiki/Q367664","display_name":"Data modeling","level":2,"score":0.2574999928474426}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2510.05318","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2510.05318","pdf_url":"https://arxiv.org/pdf/2510.05318","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2510.05318","doi":"https://doi.org/10.48550/arxiv.2510.05318","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2510.05318","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2510.05318","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2510.05318","pdf_url":"https://arxiv.org/pdf/2510.05318","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4414976110.pdf","grobid_xml":"https://content.openalex.org/works/W4414976110.grobid-xml"},"referenced_works_count":27,"referenced_works":["https://openalex.org/W2751448157","https://openalex.org/W4388092830","https://openalex.org/W4390041933","https://openalex.org/W4392019445","https://openalex.org/W4392678211","https://openalex.org/W4399115971","https://openalex.org/W4402502803","https://openalex.org/W4403703811","https://openalex.org/W4403784418","https://openalex.org/W4404344612","https://openalex.org/W4404397121","https://openalex.org/W4406840771","https://openalex.org/W4407123910","https://openalex.org/W4407759199","https://openalex.org/W4414281281","https://openalex.org/W4414903627","https://openalex.org/W4415109349","https://openalex.org/W4415348225","https://openalex.org/W4415986755","https://openalex.org/W4416050747","https://openalex.org/W4417136556","https://openalex.org/W4417148726","https://openalex.org/W4417222331","https://openalex.org/W6979315763","https://openalex.org/W7087302482","https://openalex.org/W7120088840","https://openalex.org/W7160118509"],"related_works":["https://openalex.org/W7103204440","https://openalex.org/W7133239790","https://openalex.org/W4403836156","https://openalex.org/W7125878751","https://openalex.org/W4416016791","https://openalex.org/W7223236989","https://openalex.org/W7171536026","https://openalex.org/W7172499983","https://openalex.org/W7204946433","https://openalex.org/W4410234067"],"abstract_inverted_index":{"Large":[0],"language":[1],"models":[2,86,117],"(LLMs)":[3],"have":[4],"demonstrated":[5],"remarkable":[6],"performance":[7,173],"on":[8],"single-turn":[9],"text-to-SQL":[10,225],"tasks,":[11,166],"but":[12],"real-world":[13],"database":[14,52,72],"applications":[15],"predominantly":[16],"require":[17],"multi-turn":[18,31],"interactions":[19],"to":[20,45,49,87,121,168],"handle":[21],"ambiguous":[22,154],"queries,":[23],"execution":[24],"errors,":[25],"and":[26,80,92,110,142,155,175,186,204,212],"evolving":[27],"user":[28,83,124],"requirements.":[29],"Existing":[30],"benchmarks":[32],"fall":[33],"short":[34],"by":[35,147],"treating":[36],"conversation":[37],"histories":[38],"as":[39],"static":[40],"context":[41],"or":[42,126],"limiting":[43],"evaluation":[44,101],"read-only":[46],"operations,":[47],"failing":[48],"reflect":[50],"production-grade":[51],"assistant":[53],"challenges.":[54],"We":[55],"introduce":[56],"BIRD-INTERACT,":[57],"a":[58,66,74,81,105,131],"benchmark":[59],"that":[60],"restores":[61],"this":[62],"realism":[63],"through:":[64],"(1)":[65],"comprehensive":[67,172],"interaction":[68,221],"environment":[69],"coupling":[70],"each":[71],"with":[73,179],"hierarchical":[75],"knowledge":[76],"base,":[77],"metadata":[78],"files,":[79],"function-driven":[82],"simulator,":[84],"enabling":[85],"solicit":[88],"clarifications,":[89],"retrieve":[90],"knowledge,":[91],"recover":[93],"from":[94],"errors":[95],"without":[96],"human":[97],"supervision;":[98],"(2)":[99],"two":[100],"settings":[102],"consisting":[103],"of":[104,200,219],"pre-defined":[106],"conversational":[107],"protocol":[108],"(c-Interact)":[109],"an":[111],"open-ended":[112],"agentic":[113],"setting":[114],"(a-Interact)":[115],"where":[116],"autonomously":[118],"decide":[119],"when":[120],"query":[122],"the":[123,128,136,217],"simulator":[125],"explore":[127],"environment;":[129],"(3)":[130],"challenging":[132],"task":[133,152],"suite":[134,162],"covering":[135],"full":[137],"CRUD":[138],"spectrum":[139],"for":[140,171,182,222],"business-intelligence":[141],"operational":[143],"use":[144],"cases,":[145],"guarded":[146],"executable":[148],"test":[149],"cases.":[150],"Each":[151],"features":[153],"follow-up":[156],"sub-tasks":[157],"requiring":[158],"dynamic":[159,224],"interaction.":[160],"The":[161],"comprises":[163],"BIRD-INTERACT-FULL":[164],"(600":[165],"up":[167],"11,796":[169],"interactions)":[170],"assessment,":[174],"BIRD-INTERACT-LITE":[176],"(300":[177],"tasks":[178,201],"simplified":[180],"databases)":[181],"detailed":[183],"behavioral":[184],"analysis":[185],"rapid":[187],"method":[188],"development.":[189],"Our":[190],"empirical":[191],"results":[192],"highlight":[193],"BIRD-INTERACT's":[194],"difficulty:":[195],"GPT-5":[196],"completes":[197],"only":[198],"8.67%":[199],"in":[202,206],"c-Interact":[203],"17.00%":[205],"a-Interact.":[207],"Analysis":[208],"via":[209],"memory":[210],"grafting":[211],"Interaction":[213],"Test-time":[214],"Scaling":[215],"validates":[216],"importance":[218],"effective":[220],"complex,":[223],"tasks.":[226]},"counts_by_year":[{"year":2026,"cited_by_count":24},{"year":2025,"cited_by_count":3}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2025-10-10T00:00:00"}