{"id":"https://openalex.org/W4405031556","doi":"https://doi.org/10.48550/arxiv.2411.19799","title":"INCLUDE: Evaluating Multilingual Language Understanding with Regional Knowledge","display_name":"INCLUDE: Evaluating Multilingual Language Understanding with Regional Knowledge","publication_year":2024,"publication_date":"2024-11-29","ids":{"openalex":"https://openalex.org/W4405031556","doi":"https://doi.org/10.48550/arxiv.2411.19799"},"language":"en","primary_location":{"id":"pmh:oai:pure.atira.dk:openaire/e5df06ad-9979-4159-bc66-f966b4488050","is_oa":true,"landing_page_url":"https://vbn.aau.dk/da/publications/e5df06ad-9979-4159-bc66-f966b4488050","pdf_url":null,"source":{"id":"https://openalex.org/S4306401731","display_name":"VBN Forskningsportal (Aalborg Universitet)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I891191580","host_organization_name":"Aalborg University","host_organization_lineage":["https://openalex.org/I891191580"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Data contributors 2025, INCLUDE : Evaluating Multilingual Language Understanding with Regional Knowledge. in The Thirteenth International Conference on Learning Representations. OpenReview, The Thirteenth International Conference on Learning Representations, Singapore, 24/04/2025. https://doi.org/10.48550/arXiv.2411.19799","raw_type":"info:eu-repo/semantics/publishedVersion"},"type":"article","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://vbn.aau.dk/da/publications/e5df06ad-9979-4159-bc66-f966b4488050","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5113161539","display_name":"Angelika Romanou","orcid":null},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Romanou, Angelika","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5041639724","display_name":"Negar Foroutan","orcid":"https://orcid.org/0000-0002-9640-897X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Foroutan, Negar","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5111117835","display_name":"Anna Sotnikova","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sotnikova, Anna","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5021574351","display_name":"Zeming Chen","orcid":"https://orcid.org/0000-0002-2389-6968"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Zeming","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5085857010","display_name":"Sree Harsha Nelaturu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Nelaturu, Sree Harsha","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5050856399","display_name":"Shivalika Singh","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Singh, Shivalika","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5086568118","display_name":"Rishabh Maheshwary","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Maheshwary, Rishabh","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5093547720","display_name":"Micol Altomare","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Altomare, Micol","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5118905547","display_name":"Mohamed A. Haggag","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Haggag, Mohamed A.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5115002963","display_name":"A Snegha","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"A, Snegha","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5099469319","display_name":"Alfonso Amayuelas","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Amayuelas, Alfonso","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Amirudin, Azril Hafizi","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Amirudin, Azril Hafizi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5093925862","display_name":"Viraat Aryabumi","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Aryabumi, Viraat","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5070683000","display_name":"Danylo Boiko","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Boiko, Danylo","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5063036622","display_name":"Michael Chang","orcid":"https://orcid.org/0000-0002-5153-3697"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chang, Michael","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5069738735","display_name":"Jenny Chim","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chim, Jenny","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Cohen, Gal","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Cohen, Gal","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5115002965","display_name":"Aditya Kumar Dalmia","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Dalmia, Aditya Kumar","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5115002966","display_name":"Abraham Diress","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Diress, Abraham","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Duwal, Sharad","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Duwal, Sharad","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5115002968","display_name":"Daniil Dzenhaliou","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Dzenhaliou, Daniil","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5115002969","display_name":"Daniel Fernando Erazo Florez","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Florez, Daniel Fernando Erazo","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5115002970","display_name":"Fabian Farestam","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Farestam, Fabian","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5040428208","display_name":"Joseph Marvin Imperial","orcid":"https://orcid.org/0000-0003-1073-6129"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Imperial, Joseph Marvin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102547891","display_name":"Shayekh Bin Islam","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Islam, Shayekh Bin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5115002971","display_name":"Perttu Isotalo","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Isotalo, Perttu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5115002972","display_name":"Maral Jabbarishiviari","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jabbarishiviari, Maral","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5015011965","display_name":"B\u00f6rje F. Karlsson","orcid":"https://orcid.org/0000-0001-8925-360X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Karlsson, B\u00f6rje F.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Khalilov, Eldar","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Khalilov, Eldar","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5003936190","display_name":"Christopher Klamm","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Klamm, Christopher","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5065822589","display_name":"Fajri Koto","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Koto, Fajri","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5066939448","display_name":"Dominik Krzemi\u0144ski","orcid":"https://orcid.org/0000-0003-4568-0583"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Krzemi\u0144ski, Dominik","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5036780186","display_name":"Gabriel Adriano de Melo","orcid":"https://orcid.org/0000-0001-5878-7967"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"de Melo, Gabriel Adriano","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5073430314","display_name":"Syrielle Montariol","orcid":"https://orcid.org/0000-0003-1355-8778"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Montariol, Syrielle","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100608123","display_name":"Nan Yu","orcid":"https://orcid.org/0000-0002-0177-2067"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Nan, Yiyang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5063537470","display_name":"Joel Niklaus","orcid":"https://orcid.org/0000-0002-2779-1653"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Niklaus, Joel","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5050825493","display_name":"Jekaterina Novikova","orcid":"https://orcid.org/0000-0003-4754-6126"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Novikova, Jekaterina","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5115002975","display_name":"Johan Samir Obando Ceron","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ceron, Johan Samir Obando","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5042400301","display_name":"Debjit Paul","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Paul, Debjit","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5043100830","display_name":"Esther Ploeger","orcid":"https://orcid.org/0009-0007-9525-3359"},"institutions":[{"id":"https://openalex.org/I891191580","display_name":"Aalborg University","ror":"https://ror.org/04m5j1k67","country_code":"DK","type":"education","lineage":["https://openalex.org/I891191580"]}],"countries":["DK"],"is_corresponding":false,"raw_author_name":"Ploeger, Esther","raw_affiliation_strings":["Aalborg University, The Technical Faculty of IT and Design, DK"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Aalborg University, The Technical Faculty of IT and Design, DK","institution_ids":["https://openalex.org/I891191580"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5114654176","display_name":"Jebish Purbey","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Purbey, Jebish","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5010369927","display_name":"Swati Rajwal","orcid":"https://orcid.org/0000-0002-3826-5069"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Rajwal, Swati","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5023954644","display_name":"Selvan Sunitha Ravi","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ravi, Selvan Sunitha","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5115002976","display_name":"Sara Rydell","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Rydell, Sara","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5051672173","display_name":"Roshan Santhosh","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Santhosh, Roshan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5069980465","display_name":"Drishti Sharma","orcid":"https://orcid.org/0000-0001-5908-6898"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sharma, Drishti","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5081117842","display_name":"Marjana Prifti Sk\u00ebnduli","orcid":"https://orcid.org/0000-0002-2707-1621"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Skenduli, Marjana Prifti","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5001802698","display_name":"Arshia Soltani Moakhar","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Moakhar, Arshia Soltani","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5115002978","display_name":"Bardia Soltani Moakhar","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Moakhar, Bardia Soltani","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Tamir, Ran","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tamir, Ran","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5025247914","display_name":"Ayush K Tarun","orcid":"https://orcid.org/0000-0003-1116-7597"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tarun, Ayush Kumar","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5017188403","display_name":"Azmine Toushik Wasi","orcid":"https://orcid.org/0000-0001-9509-5804"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wasi, Azmine Toushik","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5115002979","display_name":"Thenuka Ovin Weerasinghe","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Weerasinghe, Thenuka Ovin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5041505220","display_name":"Serhan Y\u0131lmaz","orcid":"https://orcid.org/0000-0003-4669-9593"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yilmaz, Serhan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5025357679","display_name":"Mike Zhang","orcid":"https://orcid.org/0000-0003-1218-5201"},"institutions":[{"id":"https://openalex.org/I891191580","display_name":"Aalborg University","ror":"https://ror.org/04m5j1k67","country_code":"DK","type":"education","lineage":["https://openalex.org/I891191580"]}],"countries":["DK"],"is_corresponding":false,"raw_author_name":"Zhang, Mike","raw_affiliation_strings":["Aalborg University, The Technical Faculty of IT and Design, DK"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Aalborg University, The Technical Faculty of IT and Design, DK","institution_ids":["https://openalex.org/I891191580"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5019167028","display_name":"Imanol Schlag","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Schlag, Imanol","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5070153551","display_name":"Marzieh Fadaee","orcid":"https://orcid.org/0000-0002-4447-1213"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Fadaee, Marzieh","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5078850040","display_name":"Sara Hooker","orcid":"https://orcid.org/0000-0002-0190-6459"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hooker, Sara","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5088410008","display_name":"Antoine Bosselut","orcid":"https://orcid.org/0000-0001-8968-9649"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Bosselut, Antoine","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":59,"corresponding_author_ids":["https://openalex.org/A5113161539"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":3,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9991999864578247,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9991999864578247,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9937000274658203,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13629","display_name":"Text Readability and Simplification","score":0.9172000288963318,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.6527889370918274},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5608155131340027},{"id":"https://openalex.org/keywords/suite","display_name":"Suite","score":0.5393826961517334},{"id":"https://openalex.org/keywords/construct","display_name":"Construct (python library)","score":0.5300165414810181},{"id":"https://openalex.org/keywords/resource","display_name":"Resource (disambiguation)","score":0.523163914680481},{"id":"https://openalex.org/keywords/multilingualism","display_name":"Multilingualism","score":0.4673251211643219},{"id":"https://openalex.org/keywords/software-deployment","display_name":"Software deployment","score":0.4512284994125366},{"id":"https://openalex.org/keywords/generative-grammar","display_name":"Generative grammar","score":0.427839994430542},{"id":"https://openalex.org/keywords/linguistics","display_name":"Linguistics","score":0.2960602343082428},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.27499282360076904},{"id":"https://openalex.org/keywords/political-science","display_name":"Political science","score":0.23817309737205505},{"id":"https://openalex.org/keywords/geography","display_name":"Geography","score":0.19347217679023743},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.07741466164588928}],"concepts":[{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.6527889370918274},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5608155131340027},{"id":"https://openalex.org/C79581498","wikidata":"https://www.wikidata.org/wiki/Q1367530","display_name":"Suite","level":2,"score":0.5393826961517334},{"id":"https://openalex.org/C2780801425","wikidata":"https://www.wikidata.org/wiki/Q5164392","display_name":"Construct (python library)","level":2,"score":0.5300165414810181},{"id":"https://openalex.org/C206345919","wikidata":"https://www.wikidata.org/wiki/Q20380951","display_name":"Resource (disambiguation)","level":2,"score":0.523163914680481},{"id":"https://openalex.org/C2780035574","wikidata":"https://www.wikidata.org/wiki/Q30081","display_name":"Multilingualism","level":2,"score":0.4673251211643219},{"id":"https://openalex.org/C105339364","wikidata":"https://www.wikidata.org/wiki/Q2297740","display_name":"Software deployment","level":2,"score":0.4512284994125366},{"id":"https://openalex.org/C39890363","wikidata":"https://www.wikidata.org/wiki/Q36108","display_name":"Generative grammar","level":2,"score":0.427839994430542},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.2960602343082428},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.27499282360076904},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.23817309737205505},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.19347217679023743},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.07741466164588928},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0}],"mesh":[],"locations_count":6,"locations":[{"id":"pmh:oai:pure.atira.dk:openaire/e5df06ad-9979-4159-bc66-f966b4488050","is_oa":true,"landing_page_url":"https://vbn.aau.dk/da/publications/e5df06ad-9979-4159-bc66-f966b4488050","pdf_url":null,"source":{"id":"https://openalex.org/S4306401731","display_name":"VBN Forskningsportal (Aalborg Universitet)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I891191580","host_organization_name":"Aalborg University","host_organization_lineage":["https://openalex.org/I891191580"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Data contributors 2025, INCLUDE : Evaluating Multilingual Language Understanding with Regional Knowledge. in The Thirteenth International Conference on Learning Representations. OpenReview, The Thirteenth International Conference on Learning Representations, Singapore, 24/04/2025. https://doi.org/10.48550/arXiv.2411.19799","raw_type":"info:eu-repo/semantics/publishedVersion"},{"id":"pmh:oai:arXiv.org:2411.19799","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2411.19799","pdf_url":"https://arxiv.org/pdf/2411.19799","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},{"id":"pmh:oai:purehost.bath.ac.uk:openaire_cris_publications/f5ba8392-c4c2-4cd8-b8fd-fb2a583fc02c","is_oa":true,"landing_page_url":"https://researchportal.bath.ac.uk/en/publications/f5ba8392-c4c2-4cd8-b8fd-fb2a583fc02c","pdf_url":"https://purehost.bath.ac.uk/ws/files/352390399/2411.19799v1.pdf","source":{"id":"https://openalex.org/S4377196294","display_name":"Pure (University of Bath)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I51601045","host_organization_name":"University of Bath","host_organization_lineage":["https://openalex.org/I51601045"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Romanou, A, Foroutan, N, Sotnikova, A, Chen, Z, Nelaturu, S H, Singh, S, Maheshwary, R, Altomare, M, Haggag, M A, Snegha, A, Amayuelas, A, Amirudin, A H, Aryabumi, V, Boiko, D, Chang, M, Chim, J, Cohen, G, Dalmia, A K, Diress, A, Duwal, S, Dzenhaliou, D, Florez, D F E, Farestam, F, Imperial, J M, Islam, S B, Isotalo, P, Jabbarishiviari, M, Karlsson, B F, Khalilov, E, Klamm, C, Koto, F, Krzemi\u0144ski, D, de Melo, G A, Montariol, S, Nan, Y, Niklaus, J, Ceron, J S O, Paul, D, Ploeger, E, Purbey, J, Rajwal, S, Ravi, S S, Rydell, S, Santhosh, R, Sharma, D, Skenduli, M P, Moakhar, A S, Moakhar, B S, Tamir, R, Tarun, A K, Wasi, A T, Weerasinghe, T O, Yilmaz, S, Zhang, M, Schlag, I, Fadaee, M, Hooker, S & Bosselut, A 2024 'Include : Evaluating Multilingual Language Understanding with Regional Knowledge' arXiv.","raw_type":"workingPaper"},{"id":"pmh:oai:infoscience.epfl.ch:20.500.14299/248414","is_oa":true,"landing_page_url":"https://infoscience.epfl.ch/handle/20.500.14299/248414","pdf_url":null,"source":{"id":"https://openalex.org/S4306400487","display_name":"Infoscience (Ecole Polytechnique F\u00e9d\u00e9rale de Lausanne)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"conference paper"},{"id":"pmh:oai:purehost.bath.ac.uk:publications/f5ba8392-c4c2-4cd8-b8fd-fb2a583fc02c","is_oa":false,"landing_page_url":"https://researchportal.bath.ac.uk/files/352390399/2411.19799v1.pdf","pdf_url":null,"source":{"id":"https://openalex.org/S4377196294","display_name":"Pure (University of Bath)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I51601045","host_organization_name":"University of Bath","host_organization_lineage":["https://openalex.org/I51601045"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Romanou, A, Foroutan, N, Sotnikova, A, Chen, Z, Nelaturu, S H, Singh, S, Maheshwary, R, Altomare, M, Haggag, M A, Snegha, A, Amayuelas, A, Amirudin, A H, Aryabumi, V, Boiko, D, Chang, M, Chim, J, Cohen, G, Dalmia, A K, Diress, A, Duwal, S, Dzenhaliou, D, Florez, D F E, Farestam, F, Imperial, J M, Islam, S B, Isotalo, P, Jabbarishiviari, M, Karlsson, B F, Khalilov, E, Klamm, C, Koto, F, Krzemi\u0144ski, D, de Melo, G A, Montariol, S, Nan, Y, Niklaus, J, Ceron, J S O, Paul, D, Ploeger, E, Purbey, J, Rajwal, S, Ravi, S S, Rydell, S, Santhosh, R, Sharma, D, Skenduli, M P, Moakhar, A S, Moakhar, B S, Tamir, R, Tarun, A K, Wasi, A T, Weerasinghe, T O, Yilmaz, S, Zhang, M, Schlag, I, Fadaee, M, Hooker, S & Bosselut, A 2024 'Include : Evaluating Multilingual Language Understanding with Regional Knowledge' arXiv.","raw_type":"workingPaper"},{"id":"doi:10.48550/arxiv.2411.19799","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2411.19799","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"pmh:oai:pure.atira.dk:openaire/e5df06ad-9979-4159-bc66-f966b4488050","is_oa":true,"landing_page_url":"https://vbn.aau.dk/da/publications/e5df06ad-9979-4159-bc66-f966b4488050","pdf_url":null,"source":{"id":"https://openalex.org/S4306401731","display_name":"VBN Forskningsportal (Aalborg Universitet)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I891191580","host_organization_name":"Aalborg University","host_organization_lineage":["https://openalex.org/I891191580"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Data contributors 2025, INCLUDE : Evaluating Multilingual Language Understanding with Regional Knowledge. in The Thirteenth International Conference on Learning Representations. OpenReview, The Thirteenth International Conference on Learning Representations, Singapore, 24/04/2025. https://doi.org/10.48550/arXiv.2411.19799","raw_type":"info:eu-repo/semantics/publishedVersion"},"sustainable_development_goals":[{"score":0.7300000190734863,"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4"}],"awards":[{"id":"https://openalex.org/G3056497733","display_name":null,"funder_award_id":"215390","funder_id":"https://openalex.org/F4320320924","funder_display_name":"Schweizerischer Nationalfonds zur F\u00f6rderung der Wissenschaftlichen Forschung"},{"id":"https://openalex.org/G8503654337","display_name":null,"funder_award_id":"PFFS-21-29","funder_id":"https://openalex.org/F4320327931","funder_display_name":"Innosuisse - Schweizerische Agentur f\u00fcr Innovationsf\u00f6rderung"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320317052","display_name":"Allen Institute for Artificial Intelligence","ror":"https://ror.org/05w520734"},{"id":"https://openalex.org/F4320320915","display_name":"\u00c9cole Polytechnique F\u00e9d\u00e9rale de Lausanne","ror":"https://ror.org/02s376052"},{"id":"https://openalex.org/F4320320924","display_name":"Schweizerischer Nationalfonds zur F\u00f6rderung der Wissenschaftlichen Forschung","ror":"https://ror.org/00yjd3n13"},{"id":"https://openalex.org/F4320327931","display_name":"Innosuisse - Schweizerische Agentur f\u00fcr Innovationsf\u00f6rderung","ror":"https://ror.org/05a2bhn71"},{"id":"https://openalex.org/F4320331518","display_name":"Centro Svizzero di Calcolo Scientifico","ror":null},{"id":"https://openalex.org/F4320331617","display_name":"National Supercomputing Centre Singapore","ror":null}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4379115841","https://openalex.org/W2083794993","https://openalex.org/W1511772879","https://openalex.org/W4394660363","https://openalex.org/W2186315912","https://openalex.org/W2588591308","https://openalex.org/W2248125223","https://openalex.org/W2910542634","https://openalex.org/W2127898439","https://openalex.org/W3082894236"],"abstract_inverted_index":{"The":[0],"performance":[1,133],"differential":[2],"of":[3,24,34,48,74,92,104,110],"large":[4],"language":[5,137],"models":[6],"(LLM)":[7],"between":[8],"languages":[9,39,53,127],"hinders":[10],"their":[11],"effective":[12],"deployment":[13],"in":[14,28,37,52,60,77,107,134],"many":[15,29,38],"regions,":[16],"inhibiting":[17],"the":[18,32,46,69,75,102,135],"potential":[19],"economic":[20],"and":[21,71,121],"societal":[22],"value":[23],"generative":[25],"AI":[26],"tools":[27],"communities.":[30],"However,":[31],"development":[33],"functional":[35],"LLMs":[36,106,131],"(\\ie,":[40],"multilingual":[41,61,79,105,130],"LLMs)":[42],"is":[43,117],"bottlenecked":[44],"by":[45],"lack":[47],"high-quality":[49],"evaluation":[50,90],"resources":[51],"other":[54],"than":[55],"English.":[56],"Moreover,":[57],"current":[58],"practices":[59],"benchmark":[62,123],"construction":[63],"often":[64],"translate":[65],"English":[66],"resources,":[67],"ignoring":[68],"regional":[70,111],"cultural":[72],"knowledge":[73],"environments":[76,138],"which":[78],"systems":[80],"would":[81,141],"be":[82,142],"used.":[83],"In":[84],"this":[85],"work,":[86],"we":[87],"construct":[88],"an":[89],"suite":[91],"197,243":[93],"QA":[94],"pairs":[95],"from":[96],"local":[97],"exam":[98],"sources":[99],"to":[100],"measure":[101],"capabilities":[103],"a":[108,118],"variety":[109],"contexts.":[112],"Our":[113],"novel":[114],"resource,":[115],"INCLUDE,":[116],"comprehensive":[119],"knowledge-":[120],"reasoning-centric":[122],"across":[124],"44":[125],"written":[126],"that":[128],"evaluates":[129],"for":[132],"actual":[136],"where":[139],"they":[140],"deployed.":[143]},"counts_by_year":[{"year":2025,"cited_by_count":3}],"updated_date":"2026-05-09T13:55:54.758798","created_date":"2024-12-05T00:00:00"}