{"id":"https://openalex.org/W7140129926","doi":"https://doi.org/10.18653/v1/2026.findings-eacl.89","title":"Garbage In, Reasoning Out? Why Benchmark Scores are Unreliable and What to Do About It","display_name":"Garbage In, Reasoning Out? Why Benchmark Scores are Unreliable and What to Do About It","publication_year":2026,"publication_date":"2026-01-01","ids":{"openalex":"https://openalex.org/W7140129926","doi":"https://doi.org/10.18653/v1/2026.findings-eacl.89"},"language":"en","primary_location":{"id":"doi:10.18653/v1/2026.findings-eacl.89","doi":"https://doi.org/10.18653/v1/2026.findings-eacl.89","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2026.findings-eacl.89","pdf_url":"https://aclanthology.org/2026.findings-eacl.89.pdf","source":{"id":"https://openalex.org/S7407087051","display_name":"Findings of the Association for Computational Linguistics: EACL","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: EACL 2026","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://aclanthology.org/2026.findings-eacl.89.pdf","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5047699997","display_name":"Seyed Mahed Mousavi","orcid":"https://orcid.org/0000-0002-7790-7730","observed_orcids":["https://orcid.org/0000-0002-7790-7730"]},"institutions":[{"id":"https://openalex.org/I193223587","display_name":"University of Trento","ror":"https://ror.org/05trd4x28","country_code":"IT","type":"education","lineage":["https://openalex.org/I193223587"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Seyed Mahed Mousavi","raw_affiliation_strings":["Signals and Interactive Systems Lab , University of Trento , Italy"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Signals and Interactive Systems Lab , University of Trento , Italy","institution_ids":["https://openalex.org/I193223587"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5120466093","display_name":"Edoardo Cecchinato","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I193223587","display_name":"University of Trento","ror":"https://ror.org/05trd4x28","country_code":"IT","type":"education","lineage":["https://openalex.org/I193223587"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Edoardo Cecchinato","raw_affiliation_strings":["Signals and Interactive Systems Lab , University of Trento , Italy"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Signals and Interactive Systems Lab , University of Trento , Italy","institution_ids":["https://openalex.org/I193223587"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5130411145","display_name":"Lucia Horn\u00edkov\u00e1","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I21449261","display_name":"Masaryk University","ror":"https://ror.org/02j46qs45","country_code":"CZ","type":"education","lineage":["https://openalex.org/I21449261"]}],"countries":["CZ"],"is_corresponding":false,"raw_author_name":"Lucia Horn\u00edkov\u00e1","raw_affiliation_strings":["Masaryk University , Czech Republic"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Masaryk University , Czech Republic","institution_ids":["https://openalex.org/I21449261"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5062879885","display_name":"Giuseppe Riccardi","orcid":"https://orcid.org/0000-0002-0739-8184","observed_orcids":["https://orcid.org/0000-0002-0739-8184"]},"institutions":[{"id":"https://openalex.org/I193223587","display_name":"University of Trento","ror":"https://ror.org/05trd4x28","country_code":"IT","type":"education","lineage":["https://openalex.org/I193223587"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Giuseppe Riccardi","raw_affiliation_strings":["Signals and Interactive Systems Lab , University of Trento , Italy"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Signals and Interactive Systems Lab , University of Trento , Italy","institution_ids":["https://openalex.org/I193223587"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.14995105,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1747","last_page":"1759"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.6317999958992004,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.6317999958992004,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.09220000356435776,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.05130000039935112,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmark-reliability","display_name":"benchmark reliability","score":0.7289999723434448},{"id":"https://openalex.org/keywords/tomis","display_name":"Tomis","score":0.9610000252723694},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.7319999933242798},{"id":"https://openalex.org/keywords/scoring-methods","display_name":"scoring methods","score":0.5730000138282776},{"id":"https://openalex.org/keywords/reasoning-assessment","display_name":"reasoning assessment","score":0.33500000834465027},{"id":"https://openalex.org/keywords/manual-annotation","display_name":"manual annotation","score":0.38999998569488525}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.660099983215332},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.4828999936580658},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4603999853134155},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.35199999809265137},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.33570000529289246},{"id":"https://openalex.org/C75403996","wikidata":"https://www.wikidata.org/wiki/Q5521979","display_name":"Garbage","level":2,"score":0.32199999690055847},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.2831000089645386},{"id":"https://openalex.org/C2776401178","wikidata":"https://www.wikidata.org/wiki/Q12050496","display_name":"Feature (linguistics)","level":2,"score":0.24210000038146973},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.23800000548362732},{"id":"https://openalex.org/C33762810","wikidata":"https://www.wikidata.org/wiki/Q461671","display_name":"Data integrity","level":2,"score":0.23600000143051147}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.18653/v1/2026.findings-eacl.89","doi":"https://doi.org/10.18653/v1/2026.findings-eacl.89","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2026.findings-eacl.89","pdf_url":"https://aclanthology.org/2026.findings-eacl.89.pdf","source":{"id":"https://openalex.org/S7407087051","display_name":"Findings of the Association for Computational Linguistics: EACL","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: EACL 2026","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.18653/v1/2026.findings-eacl.89","doi":"https://doi.org/10.18653/v1/2026.findings-eacl.89","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2026.findings-eacl.89","pdf_url":"https://aclanthology.org/2026.findings-eacl.89.pdf","source":{"id":"https://openalex.org/S7407087051","display_name":"Findings of the Association for Computational Linguistics: EACL","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: EACL 2026","raw_type":"proceedings-article"},"sustainable_development_goals":[],"sustainable_development_goals_aurora":[{"score":0.6338152885437012,"id":"https://metadata.un.org/sdg/2","display_name":"Zero hunger"}],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7140129926.pdf","grobid_xml":"https://content.openalex.org/works/W7140129926.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4416613167","https://openalex.org/W7161132819","https://openalex.org/W7154475718","https://openalex.org/W7110871625","https://openalex.org/W6910701627","https://openalex.org/W4382491841","https://openalex.org/W4416435617","https://openalex.org/W7138870598","https://openalex.org/W7204116047","https://openalex.org/W4416445891"],"abstract_inverted_index":{"We":[0],"conduct":[1],"a":[2,162],"systematic":[3,70],"audit":[4],"of":[5,142,164,189],"three":[6],"widely":[7],"used":[8],"social":[9,147],"reasoning":[10,68,148,160,191],"benchmarks,":[11],"SocialIQa,":[12],"FauxPas-EAI,":[13],"and":[14,16,24,33,43,54,73,95,118,151,179,186],"ToMi,":[15],"uncover":[17],"pervasive":[18],"flaws":[19],"in":[20,46,149],"both":[21],"benchmark":[22,47,77],"items":[23],"evaluation":[25,156,180],"methodology.Using":[26],"five":[27],"LLMs":[28],"(GPT-3,":[29],"3.5,":[30],"4,":[31],"o1,":[32],"LLaMA":[34],"3.1)":[35],"as":[36,57,59,115,161,172],"diagnostic":[37,187],"tools,":[38],"we":[39,79],"identify":[40],"structural,":[41],"semantic,":[42],"pragmatic":[44],"issues":[45],"design":[48],"(e.g.,":[49],"duplicated":[50],"items,":[51],"ambiguous":[52],"wording,":[53],"implausible":[55],"answers),":[56],"well":[58],"scoring":[60],"procedures":[61],"that":[62,81,104,121,158],"prioritize":[63],"output":[64,174],"form":[65],"over":[66],"the":[67,136,140,153],"process.Through":[69],"human":[71],"annotation":[72],"reevaluation":[74],"on":[75,135],"cleaned":[76],"subsets,":[78],"find":[80],"model":[82,105,190],"scores":[83,123],"often":[84],"improve":[85],"not":[86,96],"due":[87,89],"to":[88,90,97,110,182],"erratic":[91],"surface":[92],"wording":[93],"variations":[94,113],"improved":[98],"reasoning.In":[99],"fact,":[100],"further":[101],"analyses":[102],"show":[103],"performance":[106],"is":[107],"highly":[108],"sensitive":[109],"minor":[111],"input":[112],"such":[114],"context":[116],"availability":[117],"phrasing,":[119],"revealing":[120],"high":[122],"may":[124],"reflect":[125],"alignment":[126],"with":[127],"formatspecific":[128],"cues":[129],"rather":[130,170],"than":[131,171],"consistent":[132],"inference":[133,166],"based":[134],"input.These":[137],"findings":[138],"challenge":[139],"validity":[141],"current":[143],"benchmark-based":[144],"claims":[145],"about":[146],"LLMs,":[150],"highlight":[152],"need":[154],"for":[155],"protocols":[157],"assess":[159],"process":[163],"drawing":[165],"from":[167],"available":[168],"information,":[169],"static":[173],"selection.We":[175],"release":[176],"audited":[177],"data":[178],"tools":[181],"support":[183],"more":[184],"interpretable":[185],"assessments":[188],"1":[192],".":[193]},"counts_by_year":[],"updated_date":"2026-10-28T10:33:24.094632","created_date":"2026-03-24T00:00:00"}