{"id":"https://openalex.org/W4416033959","doi":"https://doi.org/10.18653/v1/2025.findings-emnlp.1099","title":"MASSIVE-Agents: A Benchmark for Multilingual Function-Calling in 52 Languages","display_name":"MASSIVE-Agents: A Benchmark for Multilingual Function-Calling in 52 Languages","publication_year":2025,"publication_date":"2025-01-01","ids":{"openalex":"https://openalex.org/W4416033959","doi":"https://doi.org/10.18653/v1/2025.findings-emnlp.1099"},"language":null,"primary_location":{"id":"doi:10.18653/v1/2025.findings-emnlp.1099","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.findings-emnlp.1099","pdf_url":"https://aclanthology.org/2025.findings-emnlp.1099.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: EMNLP 2025","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://aclanthology.org/2025.findings-emnlp.1099.pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5073865335","display_name":"Mayank Kulkarni","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mayank Kulkarni","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5056273253","display_name":"Vittorio Mazzia","orcid":"https://orcid.org/0000-0002-7624-1850"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Vittorio Mazzia","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5026722932","display_name":"Judith Gaspers","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Judith Gaspers","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5119968311","display_name":"Chris Hench","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chris Hench","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5086558285","display_name":"Jack FitzGerald","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jack FitzGerald","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.28632887,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"20193","last_page":"20215"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11148","display_name":"Language, Metaphor, and Cognition","score":0.3296000063419342,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T11148","display_name":"Language, Metaphor, and Cognition","score":0.3296000063419342,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12090","display_name":"Language and cultural evolution","score":0.27970001101493835,"subfield":{"id":"https://openalex.org/subfields/3316","display_name":"Cultural Studies"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12694","display_name":"Categorization, perception, and language","score":0.04610000178217888,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.43070000410079956},{"id":"https://openalex.org/keywords/feature","display_name":"Feature (linguistics)","score":0.3506999909877777},{"id":"https://openalex.org/keywords/natural-language","display_name":"Natural language","score":0.2777000069618225},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.25380000472068787}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.692300021648407},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.4860999882221222},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.45879998803138733},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.43070000410079956},{"id":"https://openalex.org/C2776401178","wikidata":"https://www.wikidata.org/wiki/Q12050496","display_name":"Feature (linguistics)","level":2,"score":0.3506999909877777},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.3393999934196472},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.3391999900341034},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.2777000069618225},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.25380000472068787},{"id":"https://openalex.org/C184337299","wikidata":"https://www.wikidata.org/wiki/Q1437428","display_name":"Semantics (computer science)","level":2,"score":0.24199999868869781}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.18653/v1/2025.findings-emnlp.1099","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.findings-emnlp.1099","pdf_url":"https://aclanthology.org/2025.findings-emnlp.1099.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: EMNLP 2025","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.18653/v1/2025.findings-emnlp.1099","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.findings-emnlp.1099","pdf_url":"https://aclanthology.org/2025.findings-emnlp.1099.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: EMNLP 2025","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4416033959.pdf","grobid_xml":"https://content.openalex.org/works/W4416033959.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"We":[0,203,216,248,262],"present":[1,62],"MASSIVE-Agents,":[2],"a":[3,110,125,182,193],"new":[4,194,265],"benchmark":[5,36,195,266],"for":[6,26,96,103,114,208],"assessing":[7],"multilingual":[8],"function":[9],"calling":[10],"across":[11,87,256],"52":[12],"languages.We":[13],"created":[14],"MASSIVE-Agents":[15],"by":[16],"cleaning":[17],"the":[18,29,63,72,115,149,185,201,205,212,225,230,234,238,264],"original":[19,226,239],"MASSIVE":[20,186,206],"dataset":[21,187,207],"and":[22,52,61,136,169,220,251,260,268],"then":[23],"reformatting":[24],"it":[25],"evaluation":[27],"within":[28],"Berkeley":[30],"Function-Calling":[31],"Leaderboard":[32,153],"(BFCL)":[33,154],"framework.The":[34],"full":[35],"comprises":[37],"47,020":[38],"samples":[39,45],"with":[40,66,71,90,130,146],"an":[41,78],"average":[42,79],"of":[43,85,112,151,184,233,253],"904":[44],"per":[46],"language,":[47],"covering":[48],"55":[49],"different":[50,134],"functions":[51],"286":[53],"arguments.We":[54],"benchmarked":[55],"21":[56,254],"models":[57,255],"using":[58,124,211],"Amazon":[59,147],"Bedrock":[60],"results":[64,121],"along":[65],"associated":[67,145],"analyses.MASSIVE-Agents":[68],"is":[69],"challenging,":[70],"top":[73],"model,":[74],"Nova":[75],"Premier,":[76],"achieving":[77],"Abstract":[80],"Syntax":[81],"Tree":[82],"(AST)":[83],"Accuracy":[84],"34.05%":[86],"all":[88],"languages,":[89,135],"performance":[91,258],"varying":[92],"significantly":[93],"from":[94,122,242],"57.37%":[95],"English":[97],"to":[98,228,244],"as":[99,101],"low":[100],"6.81%":[102],"Amharic.Some":[104],"models,":[105],"particularly":[106],"smaller":[107],"ones,":[108],"yielded":[109],"score":[111],"zero":[113],"more":[116],"difficult":[117],"languages.Additionally,":[118],"we":[119,180,196],"provide":[120],"ablations":[123,129],"custom":[126],"1-shot":[127],"prompt,":[128],"prompts":[131],"translated":[132],"into":[133,192],"comparisons":[137],"based":[138],"on":[139,224],"model":[140],"latency.":[141],"*All":[142],"authors":[143],"were":[144],"at":[148],"time":[150],"publication.Calling":[152],"(Yan":[155],"et":[156,161,173,189],"al.,":[157,162,174,190],"2024),":[158],"API-Bank":[159],"(Li":[160],"2023),":[163,168],"Nexus":[164],"Function":[165],"Calling":[166],"(team,":[167],"\u03c4":[170],"-bench":[171],"(Yao":[172],"2024).To":[175],"help":[176],"address":[177],"this":[178],"gap,":[179],"adapted":[181],"portion":[183],"(FitzGerald":[188],"2023)":[191],"call":[197],"MASSIVE-AGENTS.Our":[198],"contributions":[199],"include":[200],"following:1.":[202],"reformatted":[204],"tool":[209],"use":[210],"BFCL":[213],"framework,":[214],"2.":[215],"performed":[217],"substantial":[218],"automatic":[219],"manual":[221],"data":[222,227,267],"filtering":[223],"ensure":[229],"highest":[231],"quality":[232],"final":[235],"data,":[236],"reducing":[237],"test":[240],"set":[241],"152k":[243],"47k":[245],"utterances,":[246],"3.":[247],"conducted":[249],"benchmarking":[250],"analysis":[252],"major":[257],"tiers,":[259],"4.":[261],"release":[263],"code":[269],"publicly":[270],"1":[271],".Nexusflow.aiteam.2023.":[272]},"counts_by_year":[],"updated_date":"2026-08-21T09:56:20.448147","created_date":"2025-11-08T00:00:00"}