{"id":"https://openalex.org/W7138355761","doi":"https://doi.org/10.1609/aaai.v40i36.40262","title":"Does Question Really Matter? The Attribution of Answer Bias in LLM Evaluation","display_name":"Does Question Really Matter? The Attribution of Answer Bias in LLM Evaluation","publication_year":2026,"publication_date":"2026-03-14","ids":{"openalex":"https://openalex.org/W7138355761","doi":"https://doi.org/10.1609/aaai.v40i36.40262"},"language":"en","primary_location":{"id":"doi:10.1609/aaai.v40i36.40262","is_oa":true,"landing_page_url":"https://doi.org/10.1609/aaai.v40i36.40262","pdf_url":null,"source":{"id":"https://openalex.org/S4210191458","display_name":"Proceedings of the AAAI Conference on Artificial Intelligence","issn_l":"2159-5399","issn":["2159-5399","2374-3468"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/P4310320058","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/P4310320058"],"host_organization_lineage_names":["Association for the Advancement of Artificial Intelligence"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Artificial Intelligence","raw_type":"journal-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"diamond","oa_url":"https://doi.org/10.1609/aaai.v40i36.40262","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5129721447","display_name":"Boxi Cao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Boxi Cao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5129643979","display_name":"Ruotong Pan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ruotong Pan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5129681279","display_name":"Hongyu Lin","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hongyu Lin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5129681347","display_name":"Xianpei Han","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xianpei Han","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5129721631","display_name":"Le Sun","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Le Sun","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.20693196,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"40","issue":"36","first_page":"30130","last_page":"30138"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.7793999910354614,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.7793999910354614,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13274","display_name":"Expert finding and Q&A systems","score":0.10779999941587448,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.014000000432133675,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/debiasing","display_name":"Debiasing","score":0.8734999895095825},{"id":"https://openalex.org/keywords/popularity","display_name":"Popularity","score":0.6450999975204468},{"id":"https://openalex.org/keywords/attribution","display_name":"Attribution","score":0.6141999959945679},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.5349000096321106},{"id":"https://openalex.org/keywords/trustworthiness","display_name":"Trustworthiness","score":0.5141000151634216},{"id":"https://openalex.org/keywords/selection","display_name":"Selection (genetic algorithm)","score":0.45829999446868896},{"id":"https://openalex.org/keywords/comparability","display_name":"Comparability","score":0.45100000500679016},{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.4339999854564667},{"id":"https://openalex.org/keywords/question-answering","display_name":"Question answering","score":0.4277999997138977}],"concepts":[{"id":"https://openalex.org/C2779458634","wikidata":"https://www.wikidata.org/wiki/Q24963715","display_name":"Debiasing","level":2,"score":0.8734999895095825},{"id":"https://openalex.org/C2780586970","wikidata":"https://www.wikidata.org/wiki/Q1357284","display_name":"Popularity","level":2,"score":0.6450999975204468},{"id":"https://openalex.org/C143299363","wikidata":"https://www.wikidata.org/wiki/Q900584","display_name":"Attribution","level":2,"score":0.6141999959945679},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.5349000096321106},{"id":"https://openalex.org/C153701036","wikidata":"https://www.wikidata.org/wiki/Q659974","display_name":"Trustworthiness","level":2,"score":0.5141000151634216},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.5120000243186951},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.48510000109672546},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4650000035762787},{"id":"https://openalex.org/C81917197","wikidata":"https://www.wikidata.org/wiki/Q628760","display_name":"Selection (genetic algorithm)","level":2,"score":0.45829999446868896},{"id":"https://openalex.org/C197947376","wikidata":"https://www.wikidata.org/wiki/Q5155608","display_name":"Comparability","level":2,"score":0.45100000500679016},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.4339999854564667},{"id":"https://openalex.org/C44291984","wikidata":"https://www.wikidata.org/wiki/Q1074173","display_name":"Question answering","level":2,"score":0.4277999997138977},{"id":"https://openalex.org/C164226766","wikidata":"https://www.wikidata.org/wiki/Q7293202","display_name":"Rank (graph theory)","level":2,"score":0.3490999937057495},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.3352999985218048},{"id":"https://openalex.org/C40423286","wikidata":"https://www.wikidata.org/wiki/Q284172","display_name":"Selection bias","level":2,"score":0.33469998836517334},{"id":"https://openalex.org/C198082294","wikidata":"https://www.wikidata.org/wiki/Q3399648","display_name":"Position (finance)","level":2,"score":0.3255000114440918},{"id":"https://openalex.org/C50094484","wikidata":"https://www.wikidata.org/wiki/Q849538","display_name":"Attribution bias","level":3,"score":0.3188000023365021},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3077000081539154},{"id":"https://openalex.org/C165064840","wikidata":"https://www.wikidata.org/wiki/Q1321061","display_name":"Matching (statistics)","level":2,"score":0.295199990272522},{"id":"https://openalex.org/C75630572","wikidata":"https://www.wikidata.org/wiki/Q538904","display_name":"Applied psychology","level":1,"score":0.2851000130176544},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.2775999903678894},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.2752000093460083},{"id":"https://openalex.org/C155092808","wikidata":"https://www.wikidata.org/wiki/Q182557","display_name":"Computational linguistics","level":2,"score":0.2745000123977661},{"id":"https://openalex.org/C61797465","wikidata":"https://www.wikidata.org/wiki/Q1188986","display_name":"Term (time)","level":2,"score":0.2703999876976013},{"id":"https://openalex.org/C159447121","wikidata":"https://www.wikidata.org/wiki/Q490535","display_name":"Response bias","level":2,"score":0.26899999380111694},{"id":"https://openalex.org/C3019144022","wikidata":"https://www.wikidata.org/wiki/Q4124998","display_name":"Questions and answers","level":2,"score":0.26840001344680786},{"id":"https://openalex.org/C148220186","wikidata":"https://www.wikidata.org/wiki/Q7111912","display_name":"Outcome (game theory)","level":2,"score":0.26809999346733093},{"id":"https://openalex.org/C3018395757","wikidata":"https://www.wikidata.org/wiki/Q1379672","display_name":"Evaluation methods","level":2,"score":0.2662000060081482},{"id":"https://openalex.org/C79585631","wikidata":"https://www.wikidata.org/wiki/Q431498","display_name":"Confirmation bias","level":2,"score":0.26510000228881836},{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.2533999979496002},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.25189998745918274}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1609/aaai.v40i36.40262","is_oa":true,"landing_page_url":"https://doi.org/10.1609/aaai.v40i36.40262","pdf_url":null,"source":{"id":"https://openalex.org/S4210191458","display_name":"Proceedings of the AAAI Conference on Artificial Intelligence","issn_l":"2159-5399","issn":["2159-5399","2374-3468"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/P4310320058","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/P4310320058"],"host_organization_lineage_names":["Association for the Advancement of Artificial Intelligence"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Artificial Intelligence","raw_type":"journal-article"},{"id":"pmh:oai:ojs.aaai.org:article/40262","is_oa":true,"landing_page_url":"https://ojs.aaai.org/index.php/AAAI/article/view/40262","pdf_url":"https://ojs.aaai.org/index.php/AAAI/article/download/40262/44223","source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"2159-5399","raw_type":"info:eu-repo/semantics/article"}],"best_oa_location":{"id":"doi:10.1609/aaai.v40i36.40262","is_oa":true,"landing_page_url":"https://doi.org/10.1609/aaai.v40i36.40262","pdf_url":null,"source":{"id":"https://openalex.org/S4210191458","display_name":"Proceedings of the AAAI Conference on Artificial Intelligence","issn_l":"2159-5399","issn":["2159-5399","2374-3468"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/P4310320058","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/P4310320058"],"host_organization_lineage_names":["Association for the Advancement of Artificial Intelligence"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Artificial Intelligence","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G200577035","display_name":null,"funder_award_id":"2023-03","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Multiple-choices":[0],"question":[1],"answering":[2],"(MCQA)":[3],"has":[4],"emerged":[5],"as":[6],"one":[7],"of":[8,28,42,73,76,87,93,99,104,148],"the":[9,26,40,43,58,74,85,91,97,102,133,146],"most":[10],"popular":[11],"task":[12],"formats":[13],"for":[14,123,145],"large":[15],"language":[16],"models":[17],"(LLMs)":[18],"evaluation.":[19],"Unfortunately,":[20],"there":[21],"exist":[22],"substantial":[23],"evidence":[24],"that":[25],"evaluation":[27,44,152],"current":[29],"MCQA":[30],"benchmarks":[31],"suffers":[32],"from":[33,62],"significant":[34],"answer":[35,77,94],"bias,":[36,78,95],"which":[37],"severely":[38],"undermines":[39],"reliability":[41],"conclusions.":[45],"Specifically,":[46],"many":[47],"LLMs":[48],"achieve":[49],"performance":[50],"significantly":[51],"higher":[52],"than":[53],"random":[54],"selection":[55],"even":[56],"when":[57],"questions":[59],"are":[60],"omitted":[61],"input":[63],"information.":[64],"To":[65],"this":[66],"end,":[67],"we":[68,114],"conduct":[69],"a":[70,81,118],"systematic":[71],"investigation":[72],"attribution":[75],"and":[79,90,101,126,140],"demonstrate":[80],"strong":[82],"correlation":[83],"between":[84],"degree":[86],"data":[88],"contamination":[89,124],"severity":[92],"while":[96],"position":[98],"options":[100],"popularity":[103],"answers":[105],"have":[106],"relatively":[107],"minor":[108],"effects.":[109],"Building":[110],"on":[111],"these":[112],"insights,":[113],"further":[115],"propose":[116],"OPD,":[117],"straightforward":[119],"yet":[120],"effective":[121],"tool":[122],"detection":[125],"dataset":[127],"debiasing":[128],"without":[129],"requiring":[130],"access":[131],"to":[132],"model\u2019s":[134],"internal":[135],"training":[136],"data.":[137],"Our":[138],"findings":[139],"algorithms":[141],"provide":[142],"valuable":[143],"insights":[144],"design":[147],"future":[149],"trustworthy":[150],"LLM":[151],"protocols.":[153]},"counts_by_year":[],"updated_date":"2026-08-26T07:47:46.906454","created_date":"2026-03-18T00:00:00"}