{"id":"https://openalex.org/W4283456401","doi":"https://doi.org/10.48550/arxiv.2206.11249","title":"GEMv2: Multilingual NLG Benchmarking in a Single Line of Code","display_name":"GEMv2: Multilingual NLG Benchmarking in a Single Line of Code","publication_year":2022,"publication_date":"2022-06-22","ids":{"openalex":"https://openalex.org/W4283456401","doi":"https://doi.org/10.48550/arxiv.2206.11249"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2206.11249","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2206.11249","pdf_url":"https://arxiv.org/pdf/2206.11249","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2206.11249","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5026789962","display_name":"Sebastian Gehrmann","orcid":"https://orcid.org/0000-0002-8257-9516","observed_orcids":["https://orcid.org/0000-0002-8257-9516"]},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Gehrmann, Sebastian","raw_affiliation_strings":["Google Research , 12 Grammarly , 13"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Google Research , 12 Grammarly , 13","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5015510710","display_name":"Abhik Bhattacharjee","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I183697816","display_name":"Bangladesh University of Engineering and Technology","ror":"https://ror.org/05a1qpv97","country_code":"BD","type":"education","lineage":["https://openalex.org/I183697816"]}],"countries":["BD"],"is_corresponding":false,"raw_author_name":"Bhattacharjee, Abhik","raw_affiliation_strings":["Bangladesh University of Engineering and Technology ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Bangladesh University of Engineering and Technology ,","institution_ids":["https://openalex.org/I183697816"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5022712053","display_name":"Abinaya Mahendiran","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mahendiran, Abinaya","raw_affiliation_strings":["Mphasis NEXT Labs ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Mphasis NEXT Labs ,","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100783908","display_name":"Alex Wang","orcid":"https://orcid.org/0000-0001-8784-4801","observed_orcids":["https://orcid.org/0000-0001-8784-4801"]},"institutions":[{"id":"https://openalex.org/I57206974","display_name":"New York University","ror":"https://ror.org/0190ak572","country_code":"US","type":"education","lineage":["https://openalex.org/I57206974"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Wang, Alex","raw_affiliation_strings":["New York University , 26"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"New York University , 26","institution_ids":["https://openalex.org/I57206974"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5014990977","display_name":"Alexandros Papangelis","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I1311688040","display_name":"Amazon (United States)","ror":"https://ror.org/04mv4n011","country_code":"US","type":"company","lineage":["https://openalex.org/I1311688040"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Papangelis, Alexandros","raw_affiliation_strings":["Amazon Alexa AI ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Amazon Alexa AI ,","institution_ids":["https://openalex.org/I1311688040"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5085667632","display_name":"Aman Madaan","orcid":"https://orcid.org/0009-0009-5604-9755","observed_orcids":["https://orcid.org/0009-0009-5604-9755"]},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Madaan, Aman","raw_affiliation_strings":["Carnegie Mellon University ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University ,","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5063040968","display_name":"Angelina McMillan-Major","orcid":"https://orcid.org/0000-0003-2257-9618","observed_orcids":["https://orcid.org/0000-0003-2257-9618"]},"institutions":[{"id":"https://openalex.org/I124357947","display_name":"University of London","ror":"https://ror.org/04cw6st05","country_code":"GB","type":"education","lineage":["https://openalex.org/I124357947"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"McMillan-Major, Angelina","raw_affiliation_strings":["University of London ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of London ,","institution_ids":["https://openalex.org/I124357947"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Shvets, Anna","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shvets, Anna","raw_affiliation_strings":["Fablab in Paris by Inetum ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Fablab in Paris by Inetum ,","institution_ids":[]}]},{"author_position":"middle","author":{"id":null,"display_name":"Upadhyay, Ashish","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I522815984","display_name":"Robert Gordon University","ror":"https://ror.org/04f0qj703","country_code":"GB","type":"education","lineage":["https://openalex.org/I522815984"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Upadhyay, Ashish","raw_affiliation_strings":["Robert Gordon University , 33 Salesforce Research ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Robert Gordon University , 33 Salesforce Research ,","institution_ids":["https://openalex.org/I522815984"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5033744502","display_name":"Bingsheng Yao","orcid":"https://orcid.org/0009-0004-8329-4610","observed_orcids":["https://orcid.org/0009-0004-8329-4610"]},"institutions":[{"id":"https://openalex.org/I165799507","display_name":"Rensselaer Polytechnic Institute","ror":"https://ror.org/01rtyzb94","country_code":"US","type":"education","lineage":["https://openalex.org/I165799507"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yao, Bingsheng","raw_affiliation_strings":["Rensselaer Polytechnic Institute ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Rensselaer Polytechnic Institute ,","institution_ids":["https://openalex.org/I165799507"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5050477022","display_name":"Bryan Wilie","orcid":"https://orcid.org/0000-0001-6750-4236","observed_orcids":["https://orcid.org/0000-0001-6750-4236"]},"institutions":[{"id":"https://openalex.org/I200769079","display_name":"Hong Kong University of Science and Technology","ror":"https://ror.org/00q4vv597","country_code":"HK","type":"education","lineage":["https://openalex.org/I200769079"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Wilie, Bryan","raw_affiliation_strings":["The Hong Kong University of Science and Technology ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The Hong Kong University of Science and Technology ,","institution_ids":["https://openalex.org/I200769079"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5044250030","display_name":"Chandra Bhagavatula","orcid":"https://orcid.org/0000-0001-6264-0378","observed_orcids":["https://orcid.org/0000-0001-6264-0378"]},"institutions":[{"id":"https://openalex.org/I4210156221","display_name":"Allen Institute for Artificial Intelligence","ror":"https://ror.org/05w520734","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210156221"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Bhagavatula, Chandra","raw_affiliation_strings":["Allen Institute for AI ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Allen Institute for AI ,","institution_ids":["https://openalex.org/I4210156221"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5047526176","display_name":"Chaobin You","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I162868743","display_name":"Tianjin University","ror":"https://ror.org/012tb2g32","country_code":"CN","type":"education","lineage":["https://openalex.org/I162868743"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"You, Chaobin","raw_affiliation_strings":["Tianjin University , 42"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tianjin University , 42","institution_ids":["https://openalex.org/I162868743"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Thomson, Craig","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I200769079","display_name":"Hong Kong University of Science and Technology","ror":"https://ror.org/00q4vv597","country_code":"HK","type":"education","lineage":["https://openalex.org/I200769079"]},{"id":"https://openalex.org/I4405261251","display_name":"The Hong Kong University of Science and Technology (Guangzhou)","ror":"https://ror.org/050h0vm43","country_code":"CN","type":"education","lineage":["https://openalex.org/I4405261251"]}],"countries":["CN","HK"],"is_corresponding":false,"raw_author_name":"Thomson, Craig","raw_affiliation_strings":["HKUST , 14"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"HKUST , 14","institution_ids":["https://openalex.org/I200769079","https://openalex.org/I4405261251"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5028298718","display_name":"Cristina G\u00e2rbacea","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I27837315","display_name":"University of Michigan","ror":"https://ror.org/00jmfr291","country_code":"US","type":"education","lineage":["https://openalex.org/I27837315"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Garbacea, Cristina","raw_affiliation_strings":["University of Michigan ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Michigan ,","institution_ids":["https://openalex.org/I27837315"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5062817658","display_name":"Dakuo Wang","orcid":"https://orcid.org/0000-0001-9371-9441","observed_orcids":["https://orcid.org/0000-0001-9371-9441"]},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]},{"id":"https://openalex.org/I1341412227","display_name":"IBM (United States)","ror":"https://ror.org/05hh8d621","country_code":"US","type":"company","lineage":["https://openalex.org/I1341412227"]},{"id":"https://openalex.org/I63966007","display_name":"Massachusetts Institute of Technology","ror":"https://ror.org/042nb2s44","country_code":"US","type":"education","lineage":["https://openalex.org/I63966007"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Wang, Dakuo","raw_affiliation_strings":["MIT-IBM Watson AI Lab , 21 Masakhane , 22 Meta AI , 23 Microsoft ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"MIT-IBM Watson AI Lab , 21 Masakhane , 22 Meta AI , 23 Microsoft ,","institution_ids":["https://openalex.org/I1290206253","https://openalex.org/I1341412227","https://openalex.org/I63966007"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5007708364","display_name":"Daniel Deutsch","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I79576946","display_name":"University of Pennsylvania","ror":"https://ror.org/00b30xv10","country_code":"US","type":"education","lineage":["https://openalex.org/I79576946"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Deutsch, Daniel","raw_affiliation_strings":["University of Pennsylvania ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Pennsylvania ,","institution_ids":["https://openalex.org/I79576946"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5055232825","display_name":"Deyi Xiong","orcid":"https://orcid.org/0000-0002-2353-5038","observed_orcids":["https://orcid.org/0000-0002-2353-5038","https://orcid.org/0009-0007-6501-4496"]},"institutions":[{"id":"https://openalex.org/I162868743","display_name":"Tianjin University","ror":"https://ror.org/012tb2g32","country_code":"CN","type":"education","lineage":["https://openalex.org/I162868743"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xiong, Deyi","raw_affiliation_strings":["Tianjin University , 42"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tianjin University , 42","institution_ids":["https://openalex.org/I162868743"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5012455357","display_name":"Di Jin","orcid":"https://orcid.org/0000-0002-7445-9936","observed_orcids":["https://orcid.org/0000-0002-7445-9936"]},"institutions":[{"id":"https://openalex.org/I1311688040","display_name":"Amazon (United States)","ror":"https://ror.org/04mv4n011","country_code":"US","type":"company","lineage":["https://openalex.org/I1311688040"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jin, Di","raw_affiliation_strings":["Amazon Alexa AI ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Amazon Alexa AI ,","institution_ids":["https://openalex.org/I1311688040"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5077450332","display_name":"Dimitra Gkatzia","orcid":"https://orcid.org/0000-0001-8568-7806","observed_orcids":["https://orcid.org/0000-0001-8568-7806"]},"institutions":[{"id":"https://openalex.org/I251738","display_name":"Edinburgh Napier University","ror":"https://ror.org/03zjvnn91","country_code":"GB","type":"education","lineage":["https://openalex.org/I251738"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Gkatzia, Dimitra","raw_affiliation_strings":["Edinburgh Napier University ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Edinburgh Napier University ,","institution_ids":["https://openalex.org/I251738"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5081787254","display_name":"Dragomir Radev","orcid":"https://orcid.org/0000-0001-7830-6489","observed_orcids":["https://orcid.org/0000-0001-7830-6489"]},"institutions":[{"id":"https://openalex.org/I200769079","display_name":"Hong Kong University of Science and Technology","ror":"https://ror.org/00q4vv597","country_code":"HK","type":"education","lineage":["https://openalex.org/I200769079"]},{"id":"https://openalex.org/I4405261251","display_name":"The Hong Kong University of Science and Technology (Guangzhou)","ror":"https://ror.org/050h0vm43","country_code":"CN","type":"education","lineage":["https://openalex.org/I4405261251"]}],"countries":["CN","HK"],"is_corresponding":false,"raw_author_name":"Radev, Dragomir","raw_affiliation_strings":["HKUST , 14"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"HKUST , 14","institution_ids":["https://openalex.org/I200769079","https://openalex.org/I4405261251"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Clark, Elizabeth","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Clark, Elizabeth","raw_affiliation_strings":["Google Research , 12 Grammarly , 13"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Google Research , 12 Grammarly , 13","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5001994692","display_name":"Esin Durmus","orcid":"https://orcid.org/0009-0009-7331-8160","observed_orcids":["https://orcid.org/0009-0009-7331-8160"]},"institutions":[{"id":"https://openalex.org/I70983195","display_name":"Syracuse University","ror":"https://ror.org/025r5qe02","country_code":"US","type":"education","lineage":["https://openalex.org/I70983195"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Durmus, Esin","raw_affiliation_strings":["Stanford University , 35 Symanto Research , 36 Syracuse"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Stanford University , 35 Symanto Research , 36 Syracuse","institution_ids":["https://openalex.org/I70983195"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5074277807","display_name":"Faisal Ladhak","orcid":"https://orcid.org/0009-0009-5730-3866","observed_orcids":["https://orcid.org/0009-0009-5730-3866"]},"institutions":[{"id":"https://openalex.org/I78577930","display_name":"Columbia University","ror":"https://ror.org/00hj8s172","country_code":"US","type":"education","lineage":["https://openalex.org/I78577930"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ladhak, Faisal","raw_affiliation_strings":["Columbia University ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Columbia University ,","institution_ids":["https://openalex.org/I78577930"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5019929457","display_name":"Filip Ginter","orcid":"https://orcid.org/0000-0002-5484-6103","observed_orcids":["https://orcid.org/0000-0002-5484-6103"]},"institutions":[{"id":"https://openalex.org/I155660961","display_name":"University of Turku","ror":"https://ror.org/05vghhr25","country_code":"FI","type":"education","lineage":["https://openalex.org/I155660961"]}],"countries":["FI"],"is_corresponding":false,"raw_author_name":"Ginter, Filip","raw_affiliation_strings":["University of Turku ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Turku ,","institution_ids":["https://openalex.org/I155660961"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5085516032","display_name":"Genta Indra Winata","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I200769079","display_name":"Hong Kong University of Science and Technology","ror":"https://ror.org/00q4vv597","country_code":"HK","type":"education","lineage":["https://openalex.org/I200769079"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Winata, Genta Indra","raw_affiliation_strings":["The Hong Kong University of Science and Technology ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The Hong Kong University of Science and Technology ,","institution_ids":["https://openalex.org/I200769079"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5084459196","display_name":"Hendrik Strobelt","orcid":"https://orcid.org/0000-0002-8995-1683","observed_orcids":["https://orcid.org/0000-0002-8995-1683"]},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]},{"id":"https://openalex.org/I1341412227","display_name":"IBM (United States)","ror":"https://ror.org/05hh8d621","country_code":"US","type":"company","lineage":["https://openalex.org/I1341412227"]},{"id":"https://openalex.org/I63966007","display_name":"Massachusetts Institute of Technology","ror":"https://ror.org/042nb2s44","country_code":"US","type":"education","lineage":["https://openalex.org/I63966007"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Strobelt, Hendrik","raw_affiliation_strings":["IBM Research , 17 IIIT Hyderabad ,","MIT-IBM Watson AI Lab , 21 Masakhane , 22 Meta AI , 23 Microsoft ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM Research , 17 IIIT Hyderabad ,","institution_ids":[]},{"raw_affiliation_string":"MIT-IBM Watson AI Lab , 21 Masakhane , 22 Meta AI , 23 Microsoft ,","institution_ids":["https://openalex.org/I1290206253","https://openalex.org/I1341412227","https://openalex.org/I63966007"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Hayashi, Hiroaki","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Hayashi, Hiroaki","raw_affiliation_strings":["Carnegie Mellon University ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University ,","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5050825493","display_name":"Jekaterina Novikova","orcid":"https://orcid.org/0000-0003-4754-6126","observed_orcids":["https://orcid.org/0000-0003-4754-6126"]},"institutions":[{"id":"https://openalex.org/I32971472","display_name":"Yale University","ror":"https://ror.org/03v76x132","country_code":"US","type":"education","lineage":["https://openalex.org/I32971472"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Novikova, Jekaterina","raw_affiliation_strings":["Winterlight Labs , 51 Yale University , 52 trivago N.V .,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Winterlight Labs , 51 Yale University , 52 trivago N.V .,","institution_ids":["https://openalex.org/I32971472"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5048932608","display_name":"Jenna Kanerva","orcid":"https://orcid.org/0000-0003-4580-5366","observed_orcids":["https://orcid.org/0000-0003-4580-5366"]},"institutions":[{"id":"https://openalex.org/I155660961","display_name":"University of Turku","ror":"https://ror.org/05vghhr25","country_code":"FI","type":"education","lineage":["https://openalex.org/I155660961"]}],"countries":["FI"],"is_corresponding":false,"raw_author_name":"Kanerva, Jenna","raw_affiliation_strings":["University of Turku ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Turku ,","institution_ids":["https://openalex.org/I155660961"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5037720080","display_name":"Jenny Chim","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I153267046","display_name":"University of Kaiserslautern","ror":"https://ror.org/04zrf7b53","country_code":"DE","type":"education","lineage":["https://openalex.org/I153267046","https://openalex.org/I4387152675"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Chim, Jenny","raw_affiliation_strings":["University , 37 TU Kaiserslautern ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University , 37 TU Kaiserslautern ,","institution_ids":["https://openalex.org/I153267046"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Zhou, Jiawei","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I124357947","display_name":"University of London","ror":"https://ror.org/04cw6st05","country_code":"GB","type":"education","lineage":["https://openalex.org/I124357947"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Zhou, Jiawei","raw_affiliation_strings":["University of London ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of London ,","institution_ids":["https://openalex.org/I124357947"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5021860624","display_name":"Jordan Clive","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I200769079","display_name":"Hong Kong University of Science and Technology","ror":"https://ror.org/00q4vv597","country_code":"HK","type":"education","lineage":["https://openalex.org/I200769079"]},{"id":"https://openalex.org/I4405261251","display_name":"The Hong Kong University of Science and Technology (Guangzhou)","ror":"https://ror.org/050h0vm43","country_code":"CN","type":"education","lineage":["https://openalex.org/I4405261251"]}],"countries":["CN","HK"],"is_corresponding":false,"raw_author_name":"Clive, Jordan","raw_affiliation_strings":["HKUST , 14"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"HKUST , 14","institution_ids":["https://openalex.org/I200769079","https://openalex.org/I4405261251"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5021334672","display_name":"Joshua Maynez","orcid":"https://orcid.org/0000-0003-4948-2875","observed_orcids":["https://orcid.org/0000-0003-4948-2875"]},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Maynez, Joshua","raw_affiliation_strings":["Google Research , 12 Grammarly , 13"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Google Research , 12 Grammarly , 13","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5058954591","display_name":"Jo\u00e3o Sedoc","orcid":"https://orcid.org/0000-0001-6369-3711","observed_orcids":["https://orcid.org/0000-0001-6369-3711"]},"institutions":[{"id":"https://openalex.org/I57206974","display_name":"New York University","ror":"https://ror.org/0190ak572","country_code":"US","type":"education","lineage":["https://openalex.org/I57206974"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Sedoc, Jo\u00e3o","raw_affiliation_strings":["New York University , 26"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"New York University , 26","institution_ids":["https://openalex.org/I57206974"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5031251167","display_name":"Juraj Juraska","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I185103710","display_name":"University of California, Santa Cruz","ror":"https://ror.org/03s65by71","country_code":"US","type":"education","lineage":["https://openalex.org/I185103710"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Juraska, Juraj","raw_affiliation_strings":["University of California , Santa Cruz ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of California , Santa Cruz ,","institution_ids":["https://openalex.org/I185103710"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5061120974","display_name":"Kaustubh Dhole","orcid":"https://orcid.org/0009-0006-6907-2530","observed_orcids":["https://orcid.org/0009-0006-6907-2530","https://orcid.org/0000-0002-8875-176X"]},"institutions":[{"id":"https://openalex.org/I150468666","display_name":"Emory University","ror":"https://ror.org/03czfpz43","country_code":"US","type":"education","lineage":["https://openalex.org/I150468666"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Dhole, Kaustubh","raw_affiliation_strings":["Emory University ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Emory University ,","institution_ids":["https://openalex.org/I150468666"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5007169600","display_name":"Khyathi Raghavi Chandu","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I153267046","display_name":"University of Kaiserslautern","ror":"https://ror.org/04zrf7b53","country_code":"DE","type":"education","lineage":["https://openalex.org/I153267046","https://openalex.org/I4387152675"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Chandu, Khyathi Raghavi","raw_affiliation_strings":["University , 37 TU Kaiserslautern ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University , 37 TU Kaiserslautern ,","institution_ids":["https://openalex.org/I153267046"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5005336295","display_name":"Laura Perez-Beltrachini","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I98677209","display_name":"University of Edinburgh","ror":"https://ror.org/01nrxwf90","country_code":"GB","type":"education","lineage":["https://openalex.org/I98677209"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Perez-Beltrachini, Laura","raw_affiliation_strings":["University of Edinburgh ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Edinburgh ,","institution_ids":["https://openalex.org/I98677209"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5063685873","display_name":"Leonardo F. R. Ribeiro","orcid":"https://orcid.org/0000-0003-2639-942X","observed_orcids":["https://orcid.org/0000-0003-2639-942X"]},"institutions":[{"id":"https://openalex.org/I31512782","display_name":"Technische Universit\u00e4t Darmstadt","ror":"https://ror.org/05n911h24","country_code":"DE","type":"education","lineage":["https://openalex.org/I31512782"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Ribeiro, Leonardo F. R.","raw_affiliation_strings":["Technical University of Darmstadt ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Technical University of Darmstadt ,","institution_ids":["https://openalex.org/I31512782"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Tunstall, Lewis","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I136199984","display_name":"Harvard University","ror":"https://ror.org/03vek6s52","country_code":"US","type":"education","lineage":["https://openalex.org/I136199984"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Tunstall, Lewis","raw_affiliation_strings":["Harvard University , 15 Hugging Face ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Harvard University , 15 Hugging Face ,","institution_ids":["https://openalex.org/I136199984"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100425725","display_name":"Li Zhang","orcid":"https://orcid.org/0000-0003-2104-0194","observed_orcids":["https://orcid.org/0000-0003-2104-0194"]},"institutions":[{"id":"https://openalex.org/I79576946","display_name":"University of Pennsylvania","ror":"https://ror.org/00b30xv10","country_code":"US","type":"education","lineage":["https://openalex.org/I79576946"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Zhang, Li","raw_affiliation_strings":["University of Pennsylvania ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Pennsylvania ,","institution_ids":["https://openalex.org/I79576946"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5007386215","display_name":"Mahima Pushkarna","orcid":"https://orcid.org/0000-0002-5903-5510","observed_orcids":["https://orcid.org/0000-0002-5903-5510"]},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Pushkarna, Mahima","raw_affiliation_strings":["Google Research , 12 Grammarly , 13"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Google Research , 12 Grammarly , 13","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5061159001","display_name":"Mathias Creutz","orcid":"https://orcid.org/0000-0003-1862-4172","observed_orcids":["https://orcid.org/0000-0003-1862-4172"]},"institutions":[{"id":"https://openalex.org/I133731052","display_name":"University of Helsinki","ror":"https://ror.org/040af2s02","country_code":"FI","type":"education","lineage":["https://openalex.org/I133731052"]}],"countries":["FI"],"is_corresponding":false,"raw_author_name":"Creutz, Mathias","raw_affiliation_strings":["University of Helsinki ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Helsinki ,","institution_ids":["https://openalex.org/I133731052"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5017049205","display_name":"MICHAEL J. WHITE","orcid":"https://orcid.org/0000-0001-8828-499X","observed_orcids":["https://orcid.org/0000-0001-8828-499X"]},"institutions":[{"id":"https://openalex.org/I52357470","display_name":"The Ohio State University","ror":"https://ror.org/00rs6vg23","country_code":"US","type":"education","lineage":["https://openalex.org/I52357470"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"White, Michael","raw_affiliation_strings":["The Ohio State University ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The Ohio State University ,","institution_ids":["https://openalex.org/I52357470"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5002931312","display_name":"Mihir Kale","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kale, Mihir Sanjay","raw_affiliation_strings":["Google Research , 12 Grammarly , 13"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Google Research , 12 Grammarly , 13","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5046383980","display_name":"Moussa Kamal Eddine","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I142476485","display_name":"\u00c9cole Polytechnique","ror":"https://ror.org/05hy3tk52","country_code":"FR","type":"education","lineage":["https://openalex.org/I142476485","https://openalex.org/I4210145102"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Eddine, Moussa Kamal","raw_affiliation_strings":["\u00c9cole Polytechnique"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"\u00c9cole Polytechnique","institution_ids":["https://openalex.org/I142476485"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5075130141","display_name":"Nico Daheim","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I887968799","display_name":"RWTH Aachen University","ror":"https://ror.org/04xfq0f34","country_code":"DE","type":"education","lineage":["https://openalex.org/I887968799"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Daheim, Nico","raw_affiliation_strings":["RWTH Aachen University ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"RWTH Aachen University ,","institution_ids":["https://openalex.org/I887968799"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5109481132","display_name":"Nishant Subramani","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210156221","display_name":"Allen Institute for Artificial Intelligence","ror":"https://ror.org/05w520734","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210156221"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Subramani, Nishant","raw_affiliation_strings":["Allen Institute for AI ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Allen Institute for AI ,","institution_ids":["https://openalex.org/I4210156221"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5004829991","display_name":"Ond\u0159ej Du\u0161ek","orcid":"https://orcid.org/0000-0002-1415-1702","observed_orcids":["https://orcid.org/0000-0002-1415-1702"]},"institutions":[{"id":"https://openalex.org/I21250087","display_name":"Charles University","ror":"https://ror.org/024d6js02","country_code":"CZ","type":"education","lineage":["https://openalex.org/I21250087"]}],"countries":["CZ"],"is_corresponding":false,"raw_author_name":"Dusek, Ondrej","raw_affiliation_strings":["Charles University , 6 Chattermill ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Charles University , 6 Chattermill ,","institution_ids":["https://openalex.org/I21250087"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5086233510","display_name":"Paul Pu Liang","orcid":"https://orcid.org/0000-0001-7768-3610","observed_orcids":["https://orcid.org/0000-0001-7768-3610"]},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Liang, Paul Pu","raw_affiliation_strings":["Carnegie Mellon University ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University ,","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5087801436","display_name":"Pawan Sasanka Ammanamanchi","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I153267046","display_name":"University of Kaiserslautern","ror":"https://ror.org/04zrf7b53","country_code":"DE","type":"education","lineage":["https://openalex.org/I153267046","https://openalex.org/I4387152675"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Ammanamanchi, Pawan Sasanka","raw_affiliation_strings":["University , 37 TU Kaiserslautern ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University , 37 TU Kaiserslautern ,","institution_ids":["https://openalex.org/I153267046"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100743927","display_name":"Qi Zhu","orcid":"https://orcid.org/0000-0002-8190-8912","observed_orcids":["https://orcid.org/0000-0002-8190-8912","https://orcid.org/0000-0002-3131-2125"]},"institutions":[{"id":"https://openalex.org/I124357947","display_name":"University of London","ror":"https://ror.org/04cw6st05","country_code":"GB","type":"education","lineage":["https://openalex.org/I124357947"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Zhu, Qi","raw_affiliation_strings":["University of London ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of London ,","institution_ids":["https://openalex.org/I124357947"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5069271622","display_name":"Ratish Puduppully","orcid":"https://orcid.org/0009-0001-0857-2842","observed_orcids":["https://orcid.org/0009-0001-0857-2842"]},"institutions":[{"id":"https://openalex.org/I98677209","display_name":"University of Edinburgh","ror":"https://ror.org/01nrxwf90","country_code":"GB","type":"education","lineage":["https://openalex.org/I98677209"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Puduppully, Ratish","raw_affiliation_strings":["University of Edinburgh ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Edinburgh ,","institution_ids":["https://openalex.org/I98677209"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5082462234","display_name":"Reno Kriz","orcid":"https://orcid.org/0000-0002-0239-9989","observed_orcids":["https://orcid.org/0000-0002-0239-9989"]},"institutions":[{"id":"https://openalex.org/I145311948","display_name":"Johns Hopkins University","ror":"https://ror.org/00za53h95","country_code":"US","type":"education","lineage":["https://openalex.org/I145311948"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kriz, Reno","raw_affiliation_strings":["Johns Hopkins University ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Johns Hopkins University ,","institution_ids":["https://openalex.org/I145311948"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5090604535","display_name":"Rifat Shahriyar","orcid":"https://orcid.org/0000-0001-9540-315X","observed_orcids":["https://orcid.org/0000-0001-9540-315X"]},"institutions":[{"id":"https://openalex.org/I183697816","display_name":"Bangladesh University of Engineering and Technology","ror":"https://ror.org/05a1qpv97","country_code":"BD","type":"education","lineage":["https://openalex.org/I183697816"]}],"countries":["BD"],"is_corresponding":false,"raw_author_name":"Shahriyar, Rifat","raw_affiliation_strings":["Bangladesh University of Engineering and Technology ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Bangladesh University of Engineering and Technology ,","institution_ids":["https://openalex.org/I183697816"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5065652268","display_name":"Ronald Cardenas","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I98677209","display_name":"University of Edinburgh","ror":"https://ror.org/01nrxwf90","country_code":"GB","type":"education","lineage":["https://openalex.org/I98677209"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Cardenas, Ronald","raw_affiliation_strings":["University of Edinburgh ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Edinburgh ,","institution_ids":["https://openalex.org/I98677209"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5025882297","display_name":"Saad Mahamood","orcid":"https://orcid.org/0000-0003-2332-8749","observed_orcids":["https://orcid.org/0000-0003-2332-8749"]},"institutions":[{"id":"https://openalex.org/I124357947","display_name":"University of London","ror":"https://ror.org/04cw6st05","country_code":"GB","type":"education","lineage":["https://openalex.org/I124357947"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Mahamood, Saad","raw_affiliation_strings":["University of London ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of London ,","institution_ids":["https://openalex.org/I124357947"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5010053988","display_name":"Salomey Osei","orcid":"https://orcid.org/0000-0003-1900-3124","observed_orcids":["https://orcid.org/0000-0003-1900-3124"]},"institutions":[{"id":"https://openalex.org/I200769079","display_name":"Hong Kong University of Science and Technology","ror":"https://ror.org/00q4vv597","country_code":"HK","type":"education","lineage":["https://openalex.org/I200769079"]},{"id":"https://openalex.org/I4405261251","display_name":"The Hong Kong University of Science and Technology (Guangzhou)","ror":"https://ror.org/050h0vm43","country_code":"CN","type":"education","lineage":["https://openalex.org/I4405261251"]}],"countries":["CN","HK"],"is_corresponding":false,"raw_author_name":"Osei, Salomey","raw_affiliation_strings":["HKUST , 14"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"HKUST , 14","institution_ids":["https://openalex.org/I200769079","https://openalex.org/I4405261251"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5084836777","display_name":"Samuel Cahyawijaya","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I200769079","display_name":"Hong Kong University of Science and Technology","ror":"https://ror.org/00q4vv597","country_code":"HK","type":"education","lineage":["https://openalex.org/I200769079"]},{"id":"https://openalex.org/I4405261251","display_name":"The Hong Kong University of Science and Technology (Guangzhou)","ror":"https://ror.org/050h0vm43","country_code":"CN","type":"education","lineage":["https://openalex.org/I4405261251"]}],"countries":["CN","HK"],"is_corresponding":false,"raw_author_name":"Cahyawijaya, Samuel","raw_affiliation_strings":["HKUST , 14"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"HKUST , 14","institution_ids":["https://openalex.org/I200769079","https://openalex.org/I4405261251"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5044948242","display_name":"Sanja \u0160tajner","orcid":"https://orcid.org/0000-0002-7780-7035","observed_orcids":["https://orcid.org/0000-0002-7780-7035"]},"institutions":[{"id":"https://openalex.org/I136199984","display_name":"Harvard University","ror":"https://ror.org/03vek6s52","country_code":"US","type":"education","lineage":["https://openalex.org/I136199984"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"\u0160tajner, Sanja","raw_affiliation_strings":["Harvard University , 15 Hugging Face ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Harvard University , 15 Hugging Face ,","institution_ids":["https://openalex.org/I136199984"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5003791218","display_name":"S\u00e9bastien Montella","orcid":"https://orcid.org/0000-0002-2605-7238","observed_orcids":["https://orcid.org/0000-0002-2605-7238"]},"institutions":[{"id":"https://openalex.org/I153267046","display_name":"University of Kaiserslautern","ror":"https://ror.org/04zrf7b53","country_code":"DE","type":"education","lineage":["https://openalex.org/I153267046","https://openalex.org/I4387152675"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Montella, Sebastien","raw_affiliation_strings":["University , 37 TU Kaiserslautern ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University , 37 TU Kaiserslautern ,","institution_ids":["https://openalex.org/I153267046"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Shailza","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shailza","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5067342034","display_name":"Shailza Jolly","orcid":"https://orcid.org/0000-0002-1350-5425","observed_orcids":["https://orcid.org/0000-0002-1350-5425"]},"institutions":[{"id":"https://openalex.org/I124357947","display_name":"University of London","ror":"https://ror.org/04cw6st05","country_code":"GB","type":"education","lineage":["https://openalex.org/I124357947"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Jolly, Shailza","raw_affiliation_strings":["University of London ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of London ,","institution_ids":["https://openalex.org/I124357947"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5077016111","display_name":"Simon Mille","orcid":"https://orcid.org/0000-0002-8852-2764","observed_orcids":["https://orcid.org/0000-0002-8852-2764"]},"institutions":[{"id":"https://openalex.org/I170486558","display_name":"Universitat Pompeu Fabra","ror":"https://ror.org/04n0g0b29","country_code":"ES","type":"education","lineage":["https://openalex.org/I170486558"]}],"countries":["ES"],"is_corresponding":false,"raw_author_name":"Mille, Simon","raw_affiliation_strings":["Pompeu Fabra University , 29 Queen Mary"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Pompeu Fabra University , 29 Queen Mary","institution_ids":["https://openalex.org/I170486558"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103144863","display_name":"Tahmid Hasan","orcid":"https://orcid.org/0009-0006-2187-186X","observed_orcids":["https://orcid.org/0009-0006-2187-186X"]},"institutions":[{"id":"https://openalex.org/I183697816","display_name":"Bangladesh University of Engineering and Technology","ror":"https://ror.org/05a1qpv97","country_code":"BD","type":"education","lineage":["https://openalex.org/I183697816"]}],"countries":["BD"],"is_corresponding":false,"raw_author_name":"Hasan, Tahmid","raw_affiliation_strings":["Bangladesh University of Engineering and Technology ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Bangladesh University of Engineering and Technology ,","institution_ids":["https://openalex.org/I183697816"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101285252","display_name":"Tianhao Shen","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I162868743","display_name":"Tianjin University","ror":"https://ror.org/012tb2g32","country_code":"CN","type":"education","lineage":["https://openalex.org/I162868743"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Shen, Tianhao","raw_affiliation_strings":["Tianjin University , 42"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tianjin University , 42","institution_ids":["https://openalex.org/I162868743"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5091302921","display_name":"Tosin Adewumi","orcid":"https://orcid.org/0000-0002-5582-2031","observed_orcids":["https://orcid.org/0000-0002-5582-2031"]},"institutions":[{"id":"https://openalex.org/I190632392","display_name":"Lule\u00e5 University of Technology","ror":"https://ror.org/016st3p78","country_code":"SE","type":"education","lineage":["https://openalex.org/I190632392"]}],"countries":["SE"],"is_corresponding":false,"raw_author_name":"Adewumi, Tosin","raw_affiliation_strings":["Lule\u00e5 University of Technology ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Lule\u00e5 University of Technology ,","institution_ids":["https://openalex.org/I190632392"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5035694997","display_name":"Vikas Raunak","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I153267046","display_name":"University of Kaiserslautern","ror":"https://ror.org/04zrf7b53","country_code":"DE","type":"education","lineage":["https://openalex.org/I153267046","https://openalex.org/I4387152675"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Raunak, Vikas","raw_affiliation_strings":["University , 37 TU Kaiserslautern ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University , 37 TU Kaiserslautern ,","institution_ids":["https://openalex.org/I153267046"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5051484319","display_name":"Vipul Raheja","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I153267046","display_name":"University of Kaiserslautern","ror":"https://ror.org/04zrf7b53","country_code":"DE","type":"education","lineage":["https://openalex.org/I153267046","https://openalex.org/I4387152675"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Raheja, Vipul","raw_affiliation_strings":["University , 37 TU Kaiserslautern ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University , 37 TU Kaiserslautern ,","institution_ids":["https://openalex.org/I153267046"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5060942034","display_name":"Vitaly A. Nikolaev","orcid":"https://orcid.org/0000-0001-7054-2655","observed_orcids":["https://orcid.org/0000-0001-7054-2655"]},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Nikolaev, Vitaly","raw_affiliation_strings":["Google Research , 12 Grammarly , 13"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Google Research , 12 Grammarly , 13","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Tsai, Vivian","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Tsai, Vivian","raw_affiliation_strings":["Google Research , 12 Grammarly , 13"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Google Research , 12 Grammarly , 13","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5000126238","display_name":"Yacine Jernite","orcid":"https://orcid.org/0000-0002-8053-6862","observed_orcids":["https://orcid.org/0000-0002-8053-6862"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jernite, Yacine","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101535733","display_name":"Ying Xu","orcid":"https://orcid.org/0000-0001-7722-4385","observed_orcids":["https://orcid.org/0000-0001-7722-4385"]},"institutions":[{"id":"https://openalex.org/I27837315","display_name":"University of Michigan","ror":"https://ror.org/00jmfr291","country_code":"US","type":"education","lineage":["https://openalex.org/I27837315"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Xu, Ying","raw_affiliation_strings":["University of Michigan ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Michigan ,","institution_ids":["https://openalex.org/I27837315"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5078860840","display_name":"Yisi Sang","orcid":"https://orcid.org/0000-0002-8876-7542","observed_orcids":["https://orcid.org/0000-0002-8876-7542"]},"institutions":[{"id":"https://openalex.org/I136199984","display_name":"Harvard University","ror":"https://ror.org/03vek6s52","country_code":"US","type":"education","lineage":["https://openalex.org/I136199984"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Sang, Yisi","raw_affiliation_strings":["Harvard University , 15 Hugging Face ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Harvard University , 15 Hugging Face ,","institution_ids":["https://openalex.org/I136199984"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100386682","display_name":"Yixin Liu","orcid":"https://orcid.org/0000-0002-4309-5076","observed_orcids":["https://orcid.org/0000-0002-4309-5076"]},"institutions":[{"id":"https://openalex.org/I124357947","display_name":"University of London","ror":"https://ror.org/04cw6st05","country_code":"GB","type":"education","lineage":["https://openalex.org/I124357947"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Liu, Yixin","raw_affiliation_strings":["University of London ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of London ,","institution_ids":["https://openalex.org/I124357947"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5078812348","display_name":"Yufang Hou","orcid":"https://orcid.org/0000-0003-2897-6075","observed_orcids":["https://orcid.org/0000-0003-2897-6075"]},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Hou, Yufang","raw_affiliation_strings":["IBM Research , 17 IIIT Hyderabad ,","Tsinghua University , 43"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM Research , 17 IIIT Hyderabad ,","institution_ids":[]},{"raw_affiliation_string":"Tsinghua University , 43","institution_ids":["https://openalex.org/I99065089"]}]}],"institutions":[],"countries_distinct_count":11,"institutions_distinct_count":37,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":3,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.6640999913215637,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.6640999913215637,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.24889999628067017,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.05820000171661377,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/natural-language-generation","display_name":"natural language generation","score":0.7549999952316284},{"id":"https://openalex.org/keywords/benchmarking","display_name":"benchmarking","score":0.7630000114440918},{"id":"https://openalex.org/keywords/multilingual-dataset","display_name":"multilingual dataset","score":0.7099999785423279},{"id":"https://openalex.org/keywords/evaluation-indicators","display_name":"evaluation metrics","score":0.9039999842643738},{"id":"https://openalex.org/keywords/data-cards","display_name":"data cards","score":0.5260000228881836},{"id":"https://openalex.org/keywords/online-assessment","display_name":"online assessment","score":0.5239999890327454},{"id":"https://openalex.org/keywords/human-evaluation","display_name":"human evaluation","score":0.6190000176429749}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8618149757385254},{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.8601783514022827},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.670077383518219},{"id":"https://openalex.org/C188087704","wikidata":"https://www.wikidata.org/wiki/Q369577","display_name":"Standardization","level":2,"score":0.652036726474762},{"id":"https://openalex.org/C101468663","wikidata":"https://www.wikidata.org/wiki/Q1620158","display_name":"Modular design","level":2,"score":0.5962666869163513},{"id":"https://openalex.org/C205711294","wikidata":"https://www.wikidata.org/wiki/Q176953","display_name":"Rendering (computer graphics)","level":2,"score":0.5682143568992615},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.5162734985351562},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4736945629119873},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.46458345651626587},{"id":"https://openalex.org/C2776187449","wikidata":"https://www.wikidata.org/wiki/Q1513879","display_name":"Natural language generation","level":3,"score":0.4495568871498108},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4476171135902405},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.38265272974967957},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.33080047369003296},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.3278483748435974},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.3224068284034729},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.252444863319397},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.23458349704742432},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.0},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.0},{"id":"https://openalex.org/C162853370","wikidata":"https://www.wikidata.org/wiki/Q39809","display_name":"Marketing","level":1,"score":0.0},{"id":"https://openalex.org/C21547014","wikidata":"https://www.wikidata.org/wiki/Q1423657","display_name":"Operations management","level":1,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.0}],"mesh":[],"locations_count":6,"locations":[{"id":"pmh:oai:arXiv.org:2206.11249","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2206.11249","pdf_url":"https://arxiv.org/pdf/2206.11249","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:biblio.ufal.mff.cuni.cz:-6622357234668258372","is_oa":true,"landing_page_url":"http://hdl.handle.net/11346/BIBLIO@id=-6622357234668258372","pdf_url":null,"source":{"id":"https://openalex.org/S4306401077","display_name":"Institute of Formal and Applied Linguistics (\u00daFAL)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"info:eu-repo/semantics/other"},{"id":"pmh:oai:helda.helsinki.fi:10138/354823","is_oa":true,"landing_page_url":"http://hdl.handle.net/10138/354823","pdf_url":null,"source":{"id":"https://openalex.org/S4210213322","display_name":"Ty\u00f6v\u00e4entutkimus Vuosikirja","issn_l":"0784-1272","issn":["0784-1272","1459-7780"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":["jufo-1"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Conference contribution"},{"id":"pmh:oai:repository.hkust.edu.hk:1783.1-126587","is_oa":false,"landing_page_url":"http://www.scopus.com/record/display.url?eid=2-s2.0-85149671144&origin=inward","pdf_url":null,"source":{"id":"https://openalex.org/S4306401796","display_name":"Rare & Special e-Zone (The Hong Kong University of Science and Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I200769079","host_organization_name":"Hong Kong University of Science and Technology","host_organization_lineage":["https://openalex.org/I200769079"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Conference paper"},{"id":"pmh:oai:rgu-repository.worktribe.com:1920683","is_oa":true,"landing_page_url":"https://rgu-repository.worktribe.com/output/1920683","pdf_url":null,"source":{"id":"https://openalex.org/S4306400814","display_name":"Open Access Institutional Repository at Robert Gordon University (Robert Gordon University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I522815984","host_organization_name":"Robert Gordon University","host_organization_lineage":["https://openalex.org/I522815984"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Presentation / Conference Contribution"},{"id":"doi:10.48550/arxiv.2206.11249","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2206.11249","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2206.11249","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2206.11249","pdf_url":"https://arxiv.org/pdf/2206.11249","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"sustainable_development_goals_aurora":[{"id":"https://metadata.un.org/sdg/9","score":0.6399999856948853,"display_name":"Industry, innovation and infrastructure"}],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4238897586","https://openalex.org/W435179959","https://openalex.org/W2619091065","https://openalex.org/W2059640416","https://openalex.org/W1490753184","https://openalex.org/W2284465472","https://openalex.org/W2291782699","https://openalex.org/W1993948687","https://openalex.org/W2000169967","https://openalex.org/W2112883198"],"abstract_inverted_index":{"Evaluation":[0],"in":[1,43,103],"machine":[2],"learning":[3],"is":[4,40],"usually":[5],"informed":[6],"by":[7],"past":[8],"choices,":[9],"for":[10,86,107],"example":[11],"which":[12,47],"datasets":[13,102,109,129],"or":[14],"metrics":[15],"to":[16,57,92,126,130],"use.":[17],"This":[18,38],"standardization":[19],"enables":[20],"the":[21,29,76,131],"comparison":[22],"on":[23],"equal":[24],"footing":[25],"using":[26],"leaderboards,":[27],"but":[28],"evaluation":[30,56,66],"choices":[31],"become":[32],"sub-optimal":[33],"as":[34],"better":[35],"alternatives":[36],"arise.":[37],"problem":[39],"especially":[41],"pertinent":[42],"natural":[44],"language":[45],"generation":[46],"requires":[48],"ever-improving":[49],"suites":[50],"of":[51,75],"datasets,":[52],"metrics,":[53],"and":[54,79,89,114,120],"human":[55],"make":[58,62,123],"definitive":[59],"claims.":[60],"To":[61],"following":[63],"best":[64],"model":[65],"practices":[67],"easier,":[68],"we":[69],"introduce":[70],"GEMv2.":[71],"The":[72],"new":[73,128],"version":[74],"Generation,":[77],"Evaluation,":[78],"Metrics":[80],"Benchmark":[81],"introduces":[82],"a":[83],"modular":[84],"infrastructure":[85],"dataset,":[87],"model,":[88],"metric":[90],"developers":[91],"benefit":[93],"from":[94],"each":[95],"others":[96],"work.":[97],"GEMv2":[98],"supports":[99],"40":[100],"documented":[101],"51":[104],"languages.":[105],"Models":[106],"all":[108],"can":[110],"be":[111],"evaluated":[112],"online":[113],"our":[115],"interactive":[116],"data":[117],"card":[118],"creation":[119],"rendering":[121],"tools":[122],"it":[124],"easier":[125],"add":[127],"living":[132],"benchmark.":[133]},"counts_by_year":[{"year":2023,"cited_by_count":1},{"year":2022,"cited_by_count":2}],"updated_date":"2026-10-05T09:50:58.205359","created_date":"2025-10-10T00:00:00"}