{"id":"https://openalex.org/W4412889274","doi":"https://doi.org/10.18653/v1/2025.bea-1.3","title":"MateInfoUB: A Real-World Benchmark for Testing LLMs in Competitive, Multilingual, and Multimodal Educational Tasks","display_name":"MateInfoUB: A Real-World Benchmark for Testing LLMs in Competitive, Multilingual, and Multimodal Educational Tasks","publication_year":2025,"publication_date":"2025-01-01","ids":{"openalex":"https://openalex.org/W4412889274","doi":"https://doi.org/10.18653/v1/2025.bea-1.3"},"language":"en","primary_location":{"id":"doi:10.18653/v1/2025.bea-1.3","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.bea-1.3","pdf_url":"https://aclanthology.org/2025.bea-1.3.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 20th Workshop on Innovative Use of NLP for Building Educational Applications (BEA 2025)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://aclanthology.org/2025.bea-1.3.pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5006489572","display_name":"Marius Dumitran","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Marius Dumitran","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5119181671","display_name":"Mihnea Buca","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mihnea Buca","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5119181672","display_name":"Theodor Moroianu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Theodor Moroianu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.14592928,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"24","last_page":"37"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9897000193595886,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9897000193595886,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.8158104419708252},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6459547877311707},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.38605189323425293},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.3829735815525055},{"id":"https://openalex.org/keywords/geography","display_name":"Geography","score":0.07337871193885803}],"concepts":[{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.8158104419708252},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6459547877311707},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.38605189323425293},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3829735815525055},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.07337871193885803},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.18653/v1/2025.bea-1.3","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.bea-1.3","pdf_url":"https://aclanthology.org/2025.bea-1.3.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 20th Workshop on Innovative Use of NLP for Building Educational Applications (BEA 2025)","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.18653/v1/2025.bea-1.3","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.bea-1.3","pdf_url":"https://aclanthology.org/2025.bea-1.3.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 20th Workshop on Innovative Use of NLP for Building Educational Applications (BEA 2025)","raw_type":"proceedings-article"},"sustainable_development_goals":[{"score":0.550000011920929,"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education"}],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4412889274.pdf","grobid_xml":"https://content.openalex.org/works/W4412889274.grobid-xml"},"referenced_works_count":1,"referenced_works":["https://openalex.org/W4323570365"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2378211422","https://openalex.org/W4321353415","https://openalex.org/W2745001401","https://openalex.org/W2130974462","https://openalex.org/W2028665553","https://openalex.org/W2086519370","https://openalex.org/W3204019825"],"abstract_inverted_index":{"The":[0,92],"rapid":[1],"advancement":[2],"of":[3,48,59,71,91,111,117,145,150],"Large":[4],"Language":[5],"Models":[6],"(LLMs)":[7],"has":[8],"transformed":[9],"various":[10],"domains,":[11],"particularly":[12],"computer":[13,55],"science":[14,56],"(CS)":[15],"education.These":[16],"models":[17],"exhibit":[18],"remarkable":[19],"capabilities":[20],"in":[21,33,128,147,172,193],"code-related":[22],"tasks":[23],"and":[24,31,45,109,131,142,160,175,196],"problemsolving,":[25],"raising":[26],"questions":[27,50],"about":[28],"their":[29,99,126],"potential":[30],"limitations":[32,110],"advanced":[34],"CS":[35,129],"contexts.This":[36],"study":[37],"presents":[38],"a":[39,53],"novel":[40],"bilingual":[41],"(English-Romanian)":[42],"multimodal":[43],"(text":[44],"image)":[46],"dataset":[47,61,166,192],"multiple-choice":[49],"derived":[51],"from":[52],"high-level":[54],"competition.A":[57],"particularity":[58],"our":[60,165],"is":[62,85],"that":[63,69],"the":[64,107,115,143,148,191],"problems":[65],"are":[66,73],"conceived":[67],"such":[68],"some":[70],"them":[72,187],"easier":[74],"solved":[75],"using":[76,190],"reasoning":[77],"on":[78,95,101],"paper,":[79],"while":[80],"for":[81,183],"others":[82],"writing":[83],"code":[84],"more":[86],"efficient.We":[87],"systematically":[88],"evaluate":[89],"State":[90],"Art":[93],"LLMs":[94],"this":[96],"dataset,":[97],"analyzing":[98],"performance":[100],"theoretical":[102],"programming":[103],"tasks.Our":[104],"findings":[105],"reveal":[106],"strengths":[108],"current":[112],"LLMs,":[113],"including":[114],"influence":[116],"language":[118],"choice":[119],"(English":[120],"vs.":[121],"Romanian),":[122],"providing":[123],"insights":[124],"into":[125],"applicability":[127],"education":[130],"competition":[132],"settings.We":[133],"also":[134],"address":[135],"critical":[136],"ethical":[137],"considerations":[138],"surrounding":[139],"educational":[140,158,180],"integrity":[141],"fairness":[144],"assessments":[146],"context":[149],"LLM":[151],"usage.These":[152],"discussions":[153],"aim":[154],"to":[155,188],"inform":[156],"future":[157],"practices":[159],"policies.To":[161],"support":[162],"further":[163],"research,":[164],"will":[167],"be":[168],"made":[169],"publicly":[170],"available":[171],"both":[173],"English":[174],"Romanian.Additionally,":[176],"we":[177],"release":[178],"an":[179,194],"application":[181],"tailored":[182],"Romanian":[184],"students,":[185],"enabling":[186],"self-assess":[189],"interactive":[195],"practice-oriented":[197],"environment.":[198]},"counts_by_year":[],"updated_date":"2026-07-29T14:22:42.915294","created_date":"2025-10-10T00:00:00"}