{"id":"https://openalex.org/W4412887922","doi":"https://doi.org/10.18653/v1/2025.findings-acl.950","title":"Right Answer, Wrong Score: Uncovering the Inconsistencies of LLM Evaluation in Multiple-Choice Question Answering","display_name":"Right Answer, Wrong Score: Uncovering the Inconsistencies of LLM Evaluation in Multiple-Choice Question Answering","publication_year":2025,"publication_date":"2025-01-01","ids":{"openalex":"https://openalex.org/W4412887922","doi":"https://doi.org/10.18653/v1/2025.findings-acl.950"},"language":"en","primary_location":{"id":"doi:10.18653/v1/2025.findings-acl.950","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.findings-acl.950","pdf_url":"https://aclanthology.org/2025.findings-acl.950.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: ACL 2025","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://aclanthology.org/2025.findings-acl.950.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5106203277","display_name":"Francesco Maria Molfese","orcid":null},"institutions":[{"id":"https://openalex.org/I861853513","display_name":"Sapienza University of Rome","ror":"https://ror.org/02be6w209","country_code":"IT","type":"education","lineage":["https://openalex.org/I861853513"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Francesco Maria Molfese","raw_affiliation_strings":["Sapienza University of Rome Sapienza University of Rome Sapienza University of Rome Sapienza University of Rome Sapienza University of Rome"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Sapienza University of Rome Sapienza University of Rome Sapienza University of Rome Sapienza University of Rome Sapienza University of Rome","institution_ids":["https://openalex.org/I861853513"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Luca Moroni","orcid":null},"institutions":[{"id":"https://openalex.org/I861853513","display_name":"Sapienza University of Rome","ror":"https://ror.org/02be6w209","country_code":"IT","type":"education","lineage":["https://openalex.org/I861853513"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Luca Moroni","raw_affiliation_strings":["Sapienza University of Rome Sapienza University of Rome Sapienza University of Rome Sapienza University of Rome Sapienza University of Rome"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Sapienza University of Rome Sapienza University of Rome Sapienza University of Rome Sapienza University of Rome Sapienza University of Rome","institution_ids":["https://openalex.org/I861853513"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5119181252","display_name":"Luca Gioffr\u00e9","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Luca Gioffr\u00e9","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5078548563","display_name":"Alessandro Scir\u00e8","orcid":"https://orcid.org/0000-0002-4089-2635"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Alessandro Scir\u00e8","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5056508457","display_name":"Simone Conia","orcid":"https://orcid.org/0000-0002-6238-7816"},"institutions":[{"id":"https://openalex.org/I861853513","display_name":"Sapienza University of Rome","ror":"https://ror.org/02be6w209","country_code":"IT","type":"education","lineage":["https://openalex.org/I861853513"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Simone Conia","raw_affiliation_strings":["Sapienza University of Rome Sapienza University of Rome Sapienza University of Rome Sapienza University of Rome Sapienza University of Rome"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Sapienza University of Rome Sapienza University of Rome Sapienza University of Rome Sapienza University of Rome Sapienza University of Rome","institution_ids":["https://openalex.org/I861853513"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5026648049","display_name":"Roberto Navigli","orcid":"https://orcid.org/0000-0003-3831-9706"},"institutions":[{"id":"https://openalex.org/I861853513","display_name":"Sapienza University of Rome","ror":"https://ror.org/02be6w209","country_code":"IT","type":"education","lineage":["https://openalex.org/I861853513"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Roberto Navigli","raw_affiliation_strings":["Sapienza University of Rome Sapienza University of Rome Sapienza University of Rome Sapienza University of Rome Sapienza University of Rome"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Sapienza University of Rome Sapienza University of Rome Sapienza University of Rome Sapienza University of Rome Sapienza University of Rome","institution_ids":["https://openalex.org/I861853513"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":5.7738,"has_fulltext":true,"cited_by_count":3,"citation_normalized_percentile":{"value":0.96053545,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":91,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"18477","last_page":"18494"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13274","display_name":"Expert finding and Q&A systems","score":0.9794999957084656,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T13274","display_name":"Expert finding and Q&A systems","score":0.9794999957084656,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9046000242233276,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/question-answering","display_name":"Question answering","score":0.7323845624923706},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6683695316314697},{"id":"https://openalex.org/keywords/multiple-choice","display_name":"Multiple choice","score":0.4996209144592285},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.4141649305820465},{"id":"https://openalex.org/keywords/closed-ended-question","display_name":"Closed-ended question","score":0.41262122988700867},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.16881760954856873},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.14041370153427124}],"concepts":[{"id":"https://openalex.org/C44291984","wikidata":"https://www.wikidata.org/wiki/Q1074173","display_name":"Question answering","level":2,"score":0.7323845624923706},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6683695316314697},{"id":"https://openalex.org/C176730311","wikidata":"https://www.wikidata.org/wiki/Q867668","display_name":"Multiple choice","level":3,"score":0.4996209144592285},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.4141649305820465},{"id":"https://openalex.org/C193035329","wikidata":"https://www.wikidata.org/wiki/Q17007046","display_name":"Closed-ended question","level":2,"score":0.41262122988700867},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.16881760954856873},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.14041370153427124},{"id":"https://openalex.org/C3018023364","wikidata":"https://www.wikidata.org/wiki/Q425265","display_name":"Significant difference","level":2,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.18653/v1/2025.findings-acl.950","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.findings-acl.950","pdf_url":"https://aclanthology.org/2025.findings-acl.950.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: ACL 2025","raw_type":"proceedings-article"},{"id":"pmh:oai:iris.uniroma1.it:11573/1744349","is_oa":true,"landing_page_url":"https://hdl.handle.net/11573/1744349","pdf_url":null,"source":{"id":"https://openalex.org/S4377196107","display_name":"IRIS Research product catalog (Sapienza University of Rome)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"info:eu-repo/semantics/conferenceObject"}],"best_oa_location":{"id":"doi:10.18653/v1/2025.findings-acl.950","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.findings-acl.950","pdf_url":"https://aclanthology.org/2025.findings-acl.950.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: ACL 2025","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4412887922.pdf","grobid_xml":"https://content.openalex.org/works/W4412887922.grobid-xml"},"referenced_works_count":1,"referenced_works":["https://openalex.org/W4398192410"],"related_works":["https://openalex.org/W3107555006","https://openalex.org/W4300925379","https://openalex.org/W2391533720","https://openalex.org/W4386858563","https://openalex.org/W2033124155","https://openalex.org/W4392188973","https://openalex.org/W2031707258","https://openalex.org/W4402670313","https://openalex.org/W2782953030","https://openalex.org/W3140009346"],"abstract_inverted_index":{"One":[0],"of":[1,54,66,79,91,104],"the":[2,37,64,76,83,92,102,139,175,200],"most":[3],"widely":[4],"used":[5],"tasks":[6,22,29],"for":[7,194,202],"evaluating":[8],"Large":[9],"Language":[10],"Models":[11],"(LLMs)":[12],"is":[13,40,48],"Multiple-Choice":[14],"Question":[15],"Answering":[16],"(MCQA).":[17],"While":[18],"open-ended":[19],"question":[20,63],"answering":[21],"are":[23,125,133,159],"more":[24,203],"challenging":[25],"to":[26,34,42,45,51,62,111,161,177,184,188],"evaluate,":[27],"MCQA":[28,67,105,207],"are,":[30],"in":[31,138,174],"principle,":[32],"easier":[33],"assess,":[35],"as":[36],"model{'}s":[38],"answer":[39,93,122,136,157,179],"thought":[41],"be":[43],"simple":[44],"extract":[46],"and":[47,113,130,181,198,205],"compared":[49],"directly":[50],"a":[52,167],"set":[53],"predefined":[55],"choices.":[56,94],"However,":[57],"recent":[58],"studies":[59],"have":[60],"started":[61],"reliability":[65],"evaluation,":[68],"showing":[69],"that":[70,147],"multiple":[71],"factors":[72],"can":[73,109],"significantly":[74],"impact":[75],"reported":[77],"performance":[78],"LLMs,":[80],"especially":[81],"when":[82],"model":[84,115],"generates":[85],"free-form":[86,186],"text":[87,187],"before":[88],"selecting":[89],"one":[90],"In":[95],"this":[96],"work,":[97],"we":[98,165],"shed":[99],"light":[100],"on":[101],"inconsistencies":[103],"evaluation":[106,149,196,208],"strategies,":[107],"which":[108],"lead":[110],"inaccurate":[112],"misleading":[114],"comparisons.":[116],"We":[117],"systematically":[118],"analyze":[119],"whether":[120],"existing":[121],"extraction":[123,180],"methods":[124],"aligned":[126],"with":[127],"human":[128],"judgment,":[129],"how":[131],"they":[132],"influenced":[134],"by":[135],"constraints":[137,173],"prompt":[140,176],"across":[141],"different":[142],"domains.":[143],"Our":[144,191],"experiments":[145],"demonstrate":[146],"traditional":[148],"strategies":[150],"often":[151],"underestimate":[152],"LLM":[153],"capabilities,":[154],"while":[155],"LLM-based":[156],"extractors":[158],"prone":[160],"systematic":[162],"errors.":[163],"Moreover,":[164],"reveal":[166],"fundamental":[168],"trade-off":[169],"between":[170],"including":[171],"format":[172],"simplify":[178],"allowing":[182],"models":[183],"generate":[185],"improve":[189],"reasoning.":[190],"findings":[192],"call":[193],"standardized":[195],"methodologies":[197],"highlight":[199],"need":[201],"reliable":[204],"consistent":[206],"practices.":[209]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":1}],"updated_date":"2026-07-29T14:22:42.915294","created_date":"2025-10-10T00:00:00"}