{"id":"https://openalex.org/W7222200446","doi":"https://doi.org/10.5753/stil.2026.26539","title":"Beyond Aggregate Scores: A Diagnostic Analysis of Portuguese LLM Evaluation Suites","display_name":"Beyond Aggregate Scores: A Diagnostic Analysis of Portuguese LLM Evaluation Suites","publication_year":2026,"publication_date":"2026-10-15","ids":{"openalex":"https://openalex.org/W7222200446","doi":"https://doi.org/10.5753/stil.2026.26539"},"language":"en","primary_location":{"id":"doi:10.5753/stil.2026.26539","doi":"https://doi.org/10.5753/stil.2026.26539","is_oa":true,"landing_page_url":"https://doi.org/10.5753/stil.2026.26539","pdf_url":"https://sol.sbc.org.br/index.php/stil/article/download/44614/44377","source":{"id":"https://openalex.org/S7407105242","display_name":"Simp\u00f3sio Brasileiro de Tecnologia da Informa\u00e7\u00e3o e da Linguagem Humana (STIL)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Anais do XVII Simp\u00f3sio Brasileiro de Tecnologia da Informa\u00e7\u00e3o e da Linguagem Humana (STIL 2026)","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","is_traditional":true,"indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"bronze","oa_url":"https://sol.sbc.org.br/index.php/stil/article/download/44614/44377","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5115701227","display_name":"Gustavo Hochgraf","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I17974374","display_name":"Universidade de S\u00e3o Paulo","ror":"https://ror.org/036rp1748","country_code":"BR","type":"education","lineage":["https://openalex.org/I17974374"]}],"countries":["BR"],"is_corresponding":false,"raw_author_name":"Gustavo Hochgraf","raw_affiliation_strings":["USP"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"USP","institution_ids":["https://openalex.org/I17974374"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5115064680","display_name":"Gabriel Assis","orcid":"https://orcid.org/0009-0000-2674-0427","observed_orcids":["https://orcid.org/0009-0000-2674-0427"]},"institutions":[{"id":"https://openalex.org/I161127581","display_name":"Universidade Federal Fluminense","ror":"https://ror.org/02rjhbb08","country_code":"BR","type":"education","lineage":["https://openalex.org/I161127581"]}],"countries":["BR"],"is_corresponding":false,"raw_author_name":"Gabriel Assis","raw_affiliation_strings":["UFF"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"UFF","institution_ids":["https://openalex.org/I161127581"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5045549996","display_name":"Aline Paes","orcid":"https://orcid.org/0000-0002-9089-7303","observed_orcids":["https://orcid.org/0000-0002-9089-7303"]},"institutions":[{"id":"https://openalex.org/I161127581","display_name":"Universidade Federal Fluminense","ror":"https://ror.org/02rjhbb08","country_code":"BR","type":"education","lineage":["https://openalex.org/I161127581"]}],"countries":["BR"],"is_corresponding":false,"raw_author_name":"Aline Paes","raw_affiliation_strings":["UFF"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"UFF","institution_ids":["https://openalex.org/I161127581"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5006532598","display_name":"F\u00e1bio Gagliardi Cozman","orcid":"https://orcid.org/0000-0003-4077-4935","observed_orcids":["https://orcid.org/0000-0003-4077-4935"]},"institutions":[{"id":"https://openalex.org/I17974374","display_name":"Universidade de S\u00e3o Paulo","ror":"https://ror.org/036rp1748","country_code":"BR","type":"education","lineage":["https://openalex.org/I17974374"]}],"countries":["BR"],"is_corresponding":false,"raw_author_name":"Fabio G. Cozman","raw_affiliation_strings":["USP"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"USP","institution_ids":["https://openalex.org/I17974374"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.70329245,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"209","last_page":"217"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.6958000063896179,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.6958000063896179,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.29319998621940613,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.0019000000320374966,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/llm-evaluation","display_name":"LLM evaluation","score":0.8880000114440918},{"id":"https://openalex.org/keywords/qwen3","display_name":"Qwen3","score":0.8100000023841858},{"id":"https://openalex.org/keywords/low-resource-languages","display_name":"low-resource languages","score":0.6790000200271606}],"concepts":[{"id":"https://openalex.org/C35219183","wikidata":"https://www.wikidata.org/wiki/Q5146","display_name":"Portuguese","level":2,"score":0.8815000057220459},{"id":"https://openalex.org/C4679612","wikidata":"https://www.wikidata.org/wiki/Q866298","display_name":"Aggregate (composite)","level":2,"score":0.7425000071525574},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.6471999883651733},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.619700014591217},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.593999981880188},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.5835999846458435},{"id":"https://openalex.org/C139807058","wikidata":"https://www.wikidata.org/wiki/Q352374","display_name":"Adaptation (eye)","level":2,"score":0.5756000280380249},{"id":"https://openalex.org/C28719098","wikidata":"https://www.wikidata.org/wiki/Q44946","display_name":"Point (geometry)","level":2,"score":0.47200000286102295},{"id":"https://openalex.org/C2778334786","wikidata":"https://www.wikidata.org/wiki/Q1586270","display_name":"Variation (astronomy)","level":2,"score":0.43389999866485596},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.4120999872684479},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3292999863624573},{"id":"https://openalex.org/C2778058735","wikidata":"https://www.wikidata.org/wiki/Q4692253","display_name":"Aggregate data","level":2,"score":0.3287999927997589},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.32190001010894775},{"id":"https://openalex.org/C2778880076","wikidata":"https://www.wikidata.org/wiki/Q750553","display_name":"Brazilian Portuguese","level":3,"score":0.29829999804496765},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.29159998893737793},{"id":"https://openalex.org/C106934330","wikidata":"https://www.wikidata.org/wiki/Q1971873","display_name":"Trait","level":2,"score":0.27900001406669617},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.2773999869823456},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.26899999380111694}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.5753/stil.2026.26539","doi":"https://doi.org/10.5753/stil.2026.26539","is_oa":true,"landing_page_url":"https://doi.org/10.5753/stil.2026.26539","pdf_url":"https://sol.sbc.org.br/index.php/stil/article/download/44614/44377","source":{"id":"https://openalex.org/S7407105242","display_name":"Simp\u00f3sio Brasileiro de Tecnologia da Informa\u00e7\u00e3o e da Linguagem Humana (STIL)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Anais do XVII Simp\u00f3sio Brasileiro de Tecnologia da Informa\u00e7\u00e3o e da Linguagem Humana (STIL 2026)","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.5753/stil.2026.26539","doi":"https://doi.org/10.5753/stil.2026.26539","is_oa":true,"landing_page_url":"https://doi.org/10.5753/stil.2026.26539","pdf_url":"https://sol.sbc.org.br/index.php/stil/article/download/44614/44377","source":{"id":"https://openalex.org/S7407105242","display_name":"Simp\u00f3sio Brasileiro de Tecnologia da Informa\u00e7\u00e3o e da Linguagem Humana (STIL)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Anais do XVII Simp\u00f3sio Brasileiro de Tecnologia da Informa\u00e7\u00e3o e da Linguagem Humana (STIL 2026)","raw_type":"proceedings-article"},"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G1743900142","display_name":null,"funder_award_id":"SEI-260003/002930/2024","funder_id":"https://openalex.org/F4320322749","funder_display_name":"Funda\u00e7\u00e3o Carlos Chagas Filho de Amparo \u00e0 Pesquisa do Estado do Rio de Janeiro"},{"id":"https://openalex.org/G2100037517","display_name":null,"funder_award_id":"SEI-260003/000614/2023","funder_id":"https://openalex.org/F4320322749","funder_display_name":"Funda\u00e7\u00e3o Carlos Chagas Filho de Amparo \u00e0 Pesquisa do Estado do Rio de Janeiro"},{"id":"https://openalex.org/G2447600711","display_name":null,"funder_award_id":"260003/002930/2024","funder_id":"https://openalex.org/F4320322749","funder_display_name":"Funda\u00e7\u00e3o Carlos Chagas Filho de Amparo \u00e0 Pesquisa do Estado do Rio de Janeiro"},{"id":"https://openalex.org/G3205371473","display_name":null,"funder_award_id":"C4AI-USP","funder_id":"https://openalex.org/F4320322025","funder_display_name":"Conselho Nacional de Desenvolvimento Cient\u00edfico e Tecnol\u00f3gico"},{"id":"https://openalex.org/G3315780251","display_name":null,"funder_award_id":"SEI-260003","funder_id":"https://openalex.org/F4320322749","funder_display_name":"Funda\u00e7\u00e3o Carlos Chagas Filho de Amparo \u00e0 Pesquisa do Estado do Rio de Janeiro"},{"id":"https://openalex.org/G5395565924","display_name":null,"funder_award_id":"260003/000614/2023","funder_id":"https://openalex.org/F4320322749","funder_display_name":"Funda\u00e7\u00e3o Carlos Chagas Filho de Amparo \u00e0 Pesquisa do Estado do Rio de Janeiro"},{"id":"https://openalex.org/G5658673143","display_name":null,"funder_award_id":"2019/07665-4","funder_id":"https://openalex.org/F4320321091","funder_display_name":"Coordena\u00e7\u00e3o de Aperfei\u00e7oamento de Pessoal de N\u00edvel Superior"},{"id":"https://openalex.org/G6210502524","display_name":null,"funder_award_id":"307088/2023-5","funder_id":"https://openalex.org/F4320322025","funder_display_name":"Conselho Nacional de Desenvolvimento Cient\u00edfico e Tecnol\u00f3gico"},{"id":"https://openalex.org/G6674923580","display_name":null,"funder_award_id":"2019/07665-4","funder_id":"https://openalex.org/F4320307762","funder_display_name":"International Business Machines Corporation"},{"id":"https://openalex.org/G807454565","display_name":null,"funder_award_id":"305753/2022-3","funder_id":"https://openalex.org/F4320322025","funder_display_name":"Conselho Nacional de Desenvolvimento Cient\u00edfico e Tecnol\u00f3gico"},{"id":"https://openalex.org/G831127782","display_name":null,"funder_award_id":"#2019/07665-4","funder_id":"https://openalex.org/F4320322025","funder_display_name":"Conselho Nacional de Desenvolvimento Cient\u00edfico e Tecnol\u00f3gico"}],"funders":[{"id":"https://openalex.org/F4320307762","display_name":"International Business Machines Corporation","ror":"https://ror.org/05hh8d621"},{"id":"https://openalex.org/F4320320997","display_name":"Funda\u00e7\u00e3o de Amparo \u00e0 Pesquisa do Estado de S\u00e3o Paulo","ror":"https://ror.org/02ddkpn78"},{"id":"https://openalex.org/F4320321091","display_name":"Coordena\u00e7\u00e3o de Aperfei\u00e7oamento de Pessoal de N\u00edvel Superior","ror":"https://ror.org/00x0ma614"},{"id":"https://openalex.org/F4320322025","display_name":"Conselho Nacional de Desenvolvimento Cient\u00edfico e Tecnol\u00f3gico","ror":"https://ror.org/03swz6y49"},{"id":"https://openalex.org/F4320322749","display_name":"Funda\u00e7\u00e3o Carlos Chagas Filho de Amparo \u00e0 Pesquisa do Estado do Rio de Janeiro","ror":"https://ror.org/03kk0s825"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7222200446.pdf","grobid_xml":"https://content.openalex.org/works/W7222200446.grobid-xml"},"referenced_works_count":12,"referenced_works":["https://openalex.org/W3017961061","https://openalex.org/W3177468934","https://openalex.org/W4386148021","https://openalex.org/W4387522367","https://openalex.org/W4389520538","https://openalex.org/W4390690223","https://openalex.org/W4407926008","https://openalex.org/W4412599294","https://openalex.org/W4415148779","https://openalex.org/W4415933202","https://openalex.org/W4417452334","https://openalex.org/W4417469956"],"related_works":["https://openalex.org/W4417469956","https://openalex.org/W7143496816","https://openalex.org/W7126052303","https://openalex.org/W7164917128","https://openalex.org/W7137146072","https://openalex.org/W4415861813","https://openalex.org/W4415049870","https://openalex.org/W7215041216","https://openalex.org/W4415395499","https://openalex.org/W7143336123"],"abstract_inverted_index":{"Benchmarks":[0],"play":[1],"a":[2,28,70,76],"central":[3],"role":[4],"in":[5,138,160],"evaluating":[6,156],"large":[7],"language":[8],"models":[9],"(LLMs),":[10],"providing":[11],"standardized":[12],"comparisons":[13],"across":[14,34,104,112],"models,":[15,105],"tasks,":[16,131],"and":[17,58,116],"adaptation":[18,88,134],"strategies.":[19],"However,":[20],"aggregate":[21,150],"benchmark":[22],"scores":[23,96,151],"often":[24],"compress":[25],"performance":[26,137],"into":[27],"single":[29],"number,":[30],"obscuring":[31],"important":[32],"variation":[33],"tasks.":[35],"This":[36],"issue":[37,66],"is":[38],"especially":[39],"relevant":[40],"for":[41],"less-resourced":[42],"languages":[43],"like":[44],"Portuguese,":[45],"where":[46],"benchmarks":[47],"may":[48,125],"combine":[49],"native":[50,122],"tasks":[51,124],"with":[52,127,152],"translated":[53,130],"datasets":[54],"spanning":[55],"heterogeneous":[56],"categories":[57],"subareas.":[59],"In":[60,118],"this":[61,65],"work,":[62],"we":[63],"examine":[64],"using":[67],"PoETa":[68],"v2,":[69],"broad":[71],"Portuguese":[72,87,123,157],"evaluation":[73],"benchmark,":[74],"as":[75],"diagnostic":[77],"setting":[78],"to":[79],"evaluate":[80],"three":[81],"Qwen3":[82],"1.7B":[83],"variants":[84],"under":[85],"different":[86,110,133],"settings.":[89],"Our":[90],"results":[91],"show":[92],"that,":[93],"although":[94],"overall":[95],"differ":[97],"by":[98],"less":[99],"than":[100],"one":[101],"percentage":[102],"point":[103],"disaggregated":[106],"analyses":[107,154],"reveal":[108],"substantially":[109],"patterns":[111],"task":[113],"origin,":[114],"category,":[115],"subarea.":[117],"particular,":[119],"gains":[120],"on":[121,129],"coexist":[126],"losses":[128],"while":[132],"corpora":[135],"redistribute":[136],"distinct":[139],"ways.":[140],"Taken":[141],"together,":[142],"our":[143],"findings":[144],"highlight":[145],"the":[146,161],"importance":[147],"of":[148,163],"complementing":[149],"multi-level":[153],"when":[155],"LLMs,":[158],"particularly":[159],"context":[162],"language-specific":[164],"adaptation.":[165]},"counts_by_year":[],"updated_date":"2026-11-04T10:54:05.112534","created_date":"2026-10-16T00:00:00"}