{"id":"https://openalex.org/W4386908092","doi":"https://doi.org/10.48550/arxiv.2309.10691","title":"MINT: Evaluating LLMs in Multi-turn Interaction with Tools and Language Feedback","display_name":"MINT: Evaluating LLMs in Multi-turn Interaction with Tools and Language Feedback","publication_year":2023,"publication_date":"2023-09-19","ids":{"openalex":"https://openalex.org/W4386908092","doi":"https://doi.org/10.48550/arxiv.2309.10691"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2309.10691","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2309.10691","pdf_url":"https://arxiv.org/pdf/2309.10691","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2309.10691","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5023449213","display_name":"Xingyao Wang","orcid":"https://orcid.org/0000-0002-9502-4472","observed_orcids":["https://orcid.org/0000-0002-9502-4472"]},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Wang, Xingyao","raw_affiliation_strings":["University of Illinois Urbana-Champaign ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Illinois Urbana-Champaign ,","institution_ids":["https://openalex.org/I157725225"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100380111","display_name":"Zihan Wang","orcid":"https://orcid.org/0000-0003-4018-1603","observed_orcids":["https://orcid.org/0000-0003-4018-1603"]},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]},{"id":"https://openalex.org/I78988378","display_name":"Renmin University of China","ror":"https://ror.org/041pakw92","country_code":"CN","type":"education","lineage":["https://openalex.org/I78988378"]}],"countries":["CN","US"],"is_corresponding":false,"raw_author_name":"Wang, Zihan","raw_affiliation_strings":["Renmin University of China 1 {xingyao6 ,","University of Illinois Urbana-Champaign ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Renmin University of China 1 {xingyao6 ,","institution_ids":["https://openalex.org/I78988378"]},{"raw_affiliation_string":"University of Illinois Urbana-Champaign ,","institution_ids":["https://openalex.org/I157725225"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5075297564","display_name":"Jiateng Liu","orcid":"https://orcid.org/0000-0002-2974-5802","observed_orcids":["https://orcid.org/0000-0002-2974-5802","https://orcid.org/0009-0003-8268-2577"]},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Liu, Jiateng","raw_affiliation_strings":["University of Illinois Urbana-Champaign ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Illinois Urbana-Champaign ,","institution_ids":["https://openalex.org/I157725225"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5024379301","display_name":"Yangyi Chen","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Chen, Yangyi","raw_affiliation_strings":["University of Illinois Urbana-Champaign ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Illinois Urbana-Champaign ,","institution_ids":["https://openalex.org/I157725225"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5081648076","display_name":"Lifan Yuan","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yuan, Lifan","raw_affiliation_strings":["University of Illinois Urbana-Champaign ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Illinois Urbana-Champaign ,","institution_ids":["https://openalex.org/I157725225"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101954309","display_name":"Hao Peng","orcid":"https://orcid.org/0000-0001-8080-9595","observed_orcids":["https://orcid.org/0000-0001-8080-9595"]},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Peng, Hao","raw_affiliation_strings":["University of Illinois Urbana-Champaign ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Illinois Urbana-Champaign ,","institution_ids":["https://openalex.org/I157725225"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5103178893","display_name":"Heng Ji","orcid":"https://orcid.org/0000-0002-7954-7994","observed_orcids":["https://orcid.org/0000-0002-7954-7994"]},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ji, Heng","raw_affiliation_strings":["University of Illinois Urbana-Champaign ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Illinois Urbana-Champaign ,","institution_ids":["https://openalex.org/I157725225"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":18,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.4733999967575073,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.4733999967575073,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.17319999635219574,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.11739999800920486,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.7620000243186951},{"id":"https://openalex.org/keywords/multi-turn-interaction","display_name":"multi-turn interaction","score":0.9610000252723694},{"id":"https://openalex.org/keywords/tool-use","display_name":"tool use","score":0.8569999933242798},{"id":"https://openalex.org/keywords/natural-language-feedback","display_name":"natural language feedback","score":0.9509999752044678},{"id":"https://openalex.org/keywords/llm-benchmark","display_name":"LLM benchmark","score":0.8050000071525574},{"id":"https://openalex.org/keywords/gpt-4","display_name":"GPT-4","score":0.6119999885559082},{"id":"https://openalex.org/keywords/reasoning","display_name":"reasoning","score":0.3840000033378601},{"id":"https://openalex.org/keywords/coding","display_name":"coding","score":0.7889999747276306},{"id":"https://openalex.org/keywords/decision-making","display_name":"decision-making","score":0.9890000224113464},{"id":"https://openalex.org/keywords/reinforcement-learning-from-human-feedback","display_name":"reinforcement learning from human feedback","score":0.9850000143051147},{"id":"https://openalex.org/keywords/open-source-llms","display_name":"open-source LLMs","score":0.9210000038146973}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5953333377838135},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.5105501413345337},{"id":"https://openalex.org/C2776352735","wikidata":"https://www.wikidata.org/wiki/Q2313343","display_name":"Turn-taking","level":3,"score":0.48256105184555054},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.41685715317726135},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2857811450958252},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.2556919753551483},{"id":"https://openalex.org/C46312422","wikidata":"https://www.wikidata.org/wiki/Q11024","display_name":"Communication","level":1,"score":0.09576660394668579},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.0},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0},{"id":"https://openalex.org/C2777200299","wikidata":"https://www.wikidata.org/wiki/Q52943","display_name":"Conversation","level":2,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2309.10691","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2309.10691","pdf_url":"https://arxiv.org/pdf/2309.10691","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2309.10691","doi":"https://doi.org/10.48550/arxiv.2309.10691","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2309.10691","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2309.10691","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2309.10691","pdf_url":"https://arxiv.org/pdf/2309.10691","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"sustainable_development_goals_aurora":[{"display_name":"Peace, justice, and strong institutions","score":0.699999988079071,"id":"https://metadata.un.org/sdg/16"}],"study_designs":[],"awards":[{"id":"https://openalex.org/G1472423617","display_name":null,"funder_award_id":"FA8750-19-2-1004","funder_id":"https://openalex.org/F4320332180","funder_display_name":"Defense Advanced Research Projects Agency"},{"id":"https://openalex.org/G4713059963","display_name":null,"funder_award_id":"FA8750","funder_id":"https://openalex.org/F4320332180","funder_display_name":"Defense Advanced Research Projects Agency"},{"id":"https://openalex.org/G7559374616","display_name":null,"funder_award_id":"FA8650-23-C-7316","funder_id":"https://openalex.org/F4320332180","funder_display_name":"Defense Advanced Research Projects Agency"}],"funders":[{"id":"https://openalex.org/F4320332180","display_name":"Defense Advanced Research Projects Agency","ror":"https://ror.org/02caytj08"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4386908092.pdf","grobid_xml":"https://content.openalex.org/works/W4386908092.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2748952813","https://openalex.org/W2378211422","https://openalex.org/W2745001401","https://openalex.org/W4321353415","https://openalex.org/W2130974462","https://openalex.org/W2028665553","https://openalex.org/W2086519370","https://openalex.org/W972276598","https://openalex.org/W2087343574"],"abstract_inverted_index":{"To":[0,93],"solve":[1,78],"complex":[2],"tasks,":[3],"large":[4],"language":[5,51,91,114,163,183],"models":[6],"(LLMs)":[7],"often":[8,26],"require":[9],"multiple":[10],"rounds":[11],"of":[12,49,124,147,171,176],"interactions":[13,36,82],"with":[14,30,80,165,181,248],"the":[15,34,38,47,198],"user,":[16,39],"sometimes":[17],"assisted":[18],"by":[19,83,106,117],"external":[20,42],"tools.":[21],"However,":[22],"current":[23],"evaluation":[24,99,126,239],"protocols":[25],"emphasize":[27],"benchmark":[28,62,72],"performance":[29,166,188],"single-turn":[31,187],"exchanges,":[32],"neglecting":[33],"nuanced":[35],"among":[37],"LLMs,":[40],"and":[41,64,87,110,132,134,150,162,179,204,222],"tools,":[43],"while":[44],"also":[45],"underestimating":[46],"importance":[48],"natural":[50,90,113,182],"feedback":[52,115,209],"from":[53,160,207],"users.":[54],"These":[55],"oversights":[56],"contribute":[57],"to":[58,77,245],"discrepancies":[59],"between":[60],"research":[61,224],"evaluations":[63],"real-world":[65],"use":[66,178],"cases.":[67],"We":[68,119,215],"introduce":[69],"MINT,":[70],"a":[71,121,139,249],"that":[73],"evaluates":[74],"LLMs'":[75,227],"ability":[76],"tasks":[79],"multi-turn":[81,193,213,230,237],"(1)":[84],"using":[85],"tools":[86,105,161],"(2)":[88],"leveraging":[89],"feedback.":[92,184],"ensure":[94],"reproducibility,":[95],"we":[96],"provide":[97],"an":[98],"framework":[100],"where":[101,236],"LLMs":[102,152,157,199,247],"can":[103,218,240],"access":[104],"executing":[107],"Python":[108],"code":[109],"receive":[111],"users'":[112],"simulated":[116],"GPT-4.":[118],"repurpose":[120],"diverse":[122],"set":[123],"established":[125],"datasets":[127],"focusing":[128],"on":[129,197],"reasoning,":[130],"coding,":[131],"decision-making":[133],"carefully":[135],"curate":[136],"them":[137],"into":[138],"compact":[140],"subset":[141],"for":[142,173,233],"efficient":[143],"evaluation.":[144],"Our":[145],"analysis":[146],"20":[148],"open-":[149],"closed-source":[151],"offers":[153],"intriguing":[154],"findings.":[155],"(a)":[156],"generally":[158,211],"benefit":[159],"feedback,":[164],"gains":[167],"(absolute,":[168],"same":[169],"below)":[170],"1-8%":[172],"each":[174],"turn":[175],"tool":[177],"2-17%":[180],"(b)":[185],"Better":[186],"does":[189],"not":[190],"guarantee":[191],"better":[192],"performance.":[194],"(c)":[195],"Surprisingly,":[196],"evaluated,":[200],"supervised":[201],"instruction-finetuning":[202],"(SIFT)":[203],"reinforcement":[205],"learning":[206],"human":[208,238],"(RLHF)":[210],"hurt":[212],"capabilities.":[214],"expect":[216],"MINT":[217],"help":[219],"measure":[220],"progress":[221],"incentivize":[223],"in":[225,229],"improving":[226],"capabilities":[228],"interactions,":[231],"especially":[232],"open-source":[234],"communities":[235],"be":[241],"less":[242],"accessible":[243],"compared":[244],"commercial":[246],"larger":[250],"user":[251],"base.":[252]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":8},{"year":2024,"cited_by_count":7},{"year":2023,"cited_by_count":2}],"updated_date":"2026-10-16T09:59:22.235173","created_date":"2025-10-10T00:00:00"}