{"id":"https://openalex.org/W7204863744","doi":"https://doi.org/10.18653/2026.emnlp-main.439","title":"Large Language Models Systematically Favor Popular Options: Evidence and Mitigation Across MCQs","display_name":"Large Language Models Systematically Favor Popular Options: Evidence and Mitigation Across MCQs","publication_year":2026,"publication_date":"2026-01-01","ids":{"openalex":"https://openalex.org/W7204863744","doi":"https://doi.org/10.18653/2026.emnlp-main.439"},"language":"en","primary_location":{"id":"doi:10.18653/2026.emnlp-main.439","doi":"https://doi.org/10.18653/2026.emnlp-main.439","is_oa":true,"landing_page_url":"https://doi.org/10.18653/2026.emnlp-main.439","pdf_url":"https://aclanthology.org/2026.emnlp-main.439.pdf","source":{"id":"https://openalex.org/S4363608991","display_name":"Conference on Empirical Methods in Natural Language Processing (EMNLP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"bronze","oa_url":"https://aclanthology.org/2026.emnlp-main.439.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5133263212","display_name":"Abdelrahman Abdallah","orcid":"https://orcid.org/0000-0001-8747-4927","observed_orcids":["https://orcid.org/0000-0001-8747-4927"]},"institutions":[{"id":"https://openalex.org/I190249584","display_name":"Universit\u00e4t Innsbruck","ror":"https://ror.org/054pv6659","country_code":"AT","type":"education","lineage":["https://openalex.org/I190249584"]}],"countries":["AT"],"is_corresponding":false,"raw_author_name":"Abdelrahman Abdallah","raw_affiliation_strings":["University of Innsbruck"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Innsbruck","institution_ids":["https://openalex.org/I190249584"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133116783","display_name":"Mohammed Ali","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I190249584","display_name":"Universit\u00e4t Innsbruck","ror":"https://ror.org/054pv6659","country_code":"AT","type":"education","lineage":["https://openalex.org/I190249584"]}],"countries":["AT"],"is_corresponding":false,"raw_author_name":"Mohammed Ali","raw_affiliation_strings":["University of Innsbruck"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Innsbruck","institution_ids":["https://openalex.org/I190249584"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5007529430","display_name":"Bhawna Piryani","orcid":"https://orcid.org/0009-0005-3578-2393","observed_orcids":["https://orcid.org/0009-0005-3578-2393"]},"institutions":[{"id":"https://openalex.org/I190249584","display_name":"Universit\u00e4t Innsbruck","ror":"https://ror.org/054pv6659","country_code":"AT","type":"education","lineage":["https://openalex.org/I190249584"]}],"countries":["AT"],"is_corresponding":false,"raw_author_name":"Bhawna Piryani","raw_affiliation_strings":["University of Innsbruck"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Innsbruck","institution_ids":["https://openalex.org/I190249584"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5148842887","display_name":"Mahmoud Abdalla","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I163753206","display_name":"Chungbuk National University","ror":"https://ror.org/02wnxgj78","country_code":"KR","type":"education","lineage":["https://openalex.org/I163753206"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Mahmoud Abdalla","raw_affiliation_strings":["Chungbuk National University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Chungbuk National University","institution_ids":["https://openalex.org/I163753206"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5079733597","display_name":"Adam Jatowt","orcid":"https://orcid.org/0000-0001-7235-0665","observed_orcids":["https://orcid.org/0000-0001-7235-0665"]},"institutions":[{"id":"https://openalex.org/I190249584","display_name":"Universit\u00e4t Innsbruck","ror":"https://ror.org/054pv6659","country_code":"AT","type":"education","lineage":["https://openalex.org/I190249584"]}],"countries":["AT"],"is_corresponding":false,"raw_author_name":"Adam Jatowt","raw_affiliation_strings":["University of Innsbruck"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Innsbruck","institution_ids":["https://openalex.org/I190249584"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.64418039,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"10083","last_page":"10115"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.6284000277519226,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.6284000277519226,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.24320000410079956,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.06419999897480011,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/popularity-bias","display_name":"popularity bias","score":0.859000027179718},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.9110000133514404},{"id":"https://openalex.org/keywords/multiple-choice-questions","display_name":"multiple-choice questions","score":0.9459999799728394},{"id":"https://openalex.org/keywords/distractor-design","display_name":"distractor design","score":0.4970000088214874},{"id":"https://openalex.org/keywords/llm-evaluation","display_name":"LLM evaluation","score":0.5960000157356262}],"concepts":[{"id":"https://openalex.org/C2780586970","wikidata":"https://www.wikidata.org/wiki/Q1357284","display_name":"Popularity","level":2,"score":0.9693999886512756},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7416999936103821},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.7117000222206116},{"id":"https://openalex.org/C95713431","wikidata":"https://www.wikidata.org/wiki/Q631425","display_name":"Vulnerability (computing)","level":2,"score":0.5498999953269958},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.5449000000953674},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.5365999937057495},{"id":"https://openalex.org/C165838908","wikidata":"https://www.wikidata.org/wiki/Q736777","display_name":"Calibration","level":2,"score":0.5174000263214111},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5105000138282776},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4823000133037567},{"id":"https://openalex.org/C37736160","wikidata":"https://www.wikidata.org/wiki/Q1801315","display_name":"Adversarial system","level":2,"score":0.47049999237060547},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3882000148296356},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.3831000030040741},{"id":"https://openalex.org/C2780801425","wikidata":"https://www.wikidata.org/wiki/Q5164392","display_name":"Construct (python library)","level":2,"score":0.3643999993801117},{"id":"https://openalex.org/C44249647","wikidata":"https://www.wikidata.org/wiki/Q208498","display_name":"Confidence interval","level":2,"score":0.34450000524520874},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.33550000190734863},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.2937000095844269},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.28110000491142273},{"id":"https://openalex.org/C101454708","wikidata":"https://www.wikidata.org/wiki/Q17106019","display_name":"Standard Model (mathematical formulation)","level":3,"score":0.26600000262260437},{"id":"https://openalex.org/C12725497","wikidata":"https://www.wikidata.org/wiki/Q810247","display_name":"Baseline (sea)","level":2,"score":0.25540000200271606},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.2542000114917755}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.18653/2026.emnlp-main.439","doi":"https://doi.org/10.18653/2026.emnlp-main.439","is_oa":true,"landing_page_url":"https://doi.org/10.18653/2026.emnlp-main.439","pdf_url":"https://aclanthology.org/2026.emnlp-main.439.pdf","source":{"id":"https://openalex.org/S4363608991","display_name":"Conference on Empirical Methods in Natural Language Processing (EMNLP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2608.29257","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2608.29257","pdf_url":"https://arxiv.org/pdf/2608.29257","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2608.29257","doi":"https://doi.org/10.48550/arxiv.2608.29257","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2608.29257","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.18653/2026.emnlp-main.439","doi":"https://doi.org/10.18653/2026.emnlp-main.439","is_oa":true,"landing_page_url":"https://doi.org/10.18653/2026.emnlp-main.439","pdf_url":"https://aclanthology.org/2026.emnlp-main.439.pdf","source":{"id":"https://openalex.org/S4363608991","display_name":"Conference on Empirical Methods in Natural Language Processing (EMNLP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing","raw_type":"proceedings-article"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7204863744.pdf","grobid_xml":"https://content.openalex.org/works/W7204863744.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4386555476","https://openalex.org/W7138291094","https://openalex.org/W7138355761","https://openalex.org/W6891870617","https://openalex.org/W4406603696","https://openalex.org/W4399400301","https://openalex.org/W4416017990","https://openalex.org/W7202364925","https://openalex.org/W4416770116","https://openalex.org/W7130724978"],"abstract_inverted_index":{"Multiple-choice":[0],"questions":[1],"(MCQs)":[2],"are":[3,86],"a":[4,34,63,109,117,133],"standard":[5],"format":[6],"for":[7,53,137],"evaluating":[8],"large":[9],"language":[10],"models":[11,93],"(LLMs),":[12],"yet":[13],"the":[14,75,90,101],"popularity":[15,38,72,118,166],"of":[16,100],"answer":[17,77],"options":[18,28],"can":[19],"confound":[20],"evaluation.Modern":[21],"LLMs":[22,148],"systematically":[23,56],"prefer":[24],"popular":[25,31,54,88,95],"but":[26,96],"incorrect":[27],"over":[29],"less":[30],"correct":[32,76,91],"ones,":[33],"vulnerability":[35],"we":[36,60,106],"call":[37],"bias.This":[39],"pattern":[40],"aligns":[41],"with":[42,65,156],"confidence":[43,46],"miscalibration:":[44],"model":[45,121],"remains":[47],"high":[48],"even":[49],"as":[50],"accuracy":[51,157],"collapses":[52],"options.To":[55],"isolate":[57],"this":[58,104],"phenomenon,":[59],"introduce":[61],"PopMCQ,":[62],"benchmark":[64],"six":[66],"controlled":[67],"strategies":[68],"that":[69,113],"vary":[70],"option":[71],"while":[73],"keeping":[74],"fixed.In":[78],"our":[79],"most":[80],"adversarial":[81],"setting,":[82],"where":[83],"all":[84],"distractors":[85],"more":[87],"than":[89],"option,":[92],"choose":[94],"wrong":[97],"answers":[98],"66%":[99],"time.To":[102],"mitigate":[103],"bias,":[105],"propose":[107],"PopDebias,":[108],"lightweight":[110],"inference-time":[111],"correction":[112],"estimates":[114],"and":[115,140],"removes":[116],"prior":[119],"from":[120],"predictions.It":[122],"requires":[123],"no":[124],"fine-tuning,":[125],"is":[126],"label-free":[127],"at":[128],"test":[129],"time":[130],"(using":[131],"only":[132],"small":[134],"calibration":[135],"split":[136],"parameter":[138],"fitting),":[139],"adds":[141],"negligible":[142],"computational":[143],"cost.Experiments":[144],"on":[145],"22":[146],"open-source":[147],"(0.5B":[149],"to":[150,160],"32B":[151],"parameters)":[152],"show":[153],"consistent":[154],"improvements,":[155],"gains":[158],"up":[159],"54.1":[161],"percentage":[162],"points":[163],"under":[164],"strong":[165],"pressure":[167],"1":[168],".-1.0":[169],"-0.50":[170,216],"0":[171,217],"0.25":[172,218],"Spearman":[173],"=":[174],"0.5":[175],"(visual":[176],"guide)":[177],"DeepSeek-V2-Lite":[178],"EntityQuestions":[179],"-0.28":[180],"-0.90":[181,188,204],"-0.70":[182],"-0.52":[183],"0.23":[184],"-0.94":[185],"NQ":[186],"-0.32":[187],"-0.72":[189],"-0.55":[190],"0.15":[191],"-0.97":[192],"MuSiQue":[193],"-0.23":[194],"-0.89":[195],"-0.66":[196],"-0.490.19":[197],"-0.95":[198],"Bias":[199],"zone":[200],"Alignment":[201],"WebQuestions":[202],"-0.34":[203],"-0.68":[205],"-0.53":[206],"0.21":[207],"-0.98":[208],"S1":[209],"S2":[210],"S3":[211],"S4":[212],"S5":[213],"S6":[214],"-1.0":[215],"SpearmanBase":[219],"Pop.Trap":[220],"Grad.":[221]},"counts_by_year":[],"updated_date":"2026-11-04T10:54:05.112534","created_date":"2026-09-02T00:00:00"}