{"id":"https://openalex.org/W7168343197","doi":"https://doi.org/10.1145/3805712.3809901","title":"The LLM Effect on IR Benchmarks: A Meta-Analysis of Effectiveness, Baselines, and Contamination","display_name":"The LLM Effect on IR Benchmarks: A Meta-Analysis of Effectiveness, Baselines, and Contamination","publication_year":2026,"publication_date":"2026-07-15","ids":{"openalex":"https://openalex.org/W7168343197","doi":"https://doi.org/10.1145/3805712.3809901"},"language":null,"primary_location":{"id":"doi:10.1145/3805712.3809901","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3805712.3809901","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 49th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},"type":"review","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5140890492","display_name":"Moritz Staudinger","orcid":"https://orcid.org/0000-0002-5164-2690"},"institutions":[{"id":"https://openalex.org/I145847075","display_name":"TU Wien","ror":"https://ror.org/04d836q62","country_code":"AT","type":"education","lineage":["https://openalex.org/I145847075"]}],"countries":["AT"],"is_corresponding":false,"raw_author_name":"Moritz Staudinger","raw_affiliation_strings":["TU Wien, Vienna, Austria"],"raw_orcid":"https://orcid.org/0000-0002-5164-2690","affiliations":[{"raw_affiliation_string":"TU Wien, Vienna, Austria","institution_ids":["https://openalex.org/I145847075"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5140875175","display_name":"Wojciech Kusa","orcid":"https://orcid.org/0000-0003-4420-4147"},"institutions":[{"id":"https://openalex.org/I4210136276","display_name":"NASK National Research Institute","ror":"https://ror.org/03ajsaw82","country_code":"PL","type":"facility","lineage":["https://openalex.org/I4210136276"]}],"countries":["PL"],"is_corresponding":false,"raw_author_name":"Wojciech Kusa","raw_affiliation_strings":["NASK - National Research Institute, Warsaw, Poland"],"raw_orcid":"https://orcid.org/0000-0003-4420-4147","affiliations":[{"raw_affiliation_string":"NASK - National Research Institute, Warsaw, Poland","institution_ids":["https://openalex.org/I4210136276"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5140820079","display_name":"Allan Hanbury","orcid":"https://orcid.org/0000-0002-7149-5843"},"institutions":[{"id":"https://openalex.org/I145847075","display_name":"TU Wien","ror":"https://ror.org/04d836q62","country_code":"AT","type":"education","lineage":["https://openalex.org/I145847075"]}],"countries":["AT"],"is_corresponding":false,"raw_author_name":"Allan Hanbury","raw_affiliation_strings":["TU Wien, Vienna, Austria"],"raw_orcid":"https://orcid.org/0000-0002-7149-5843","affiliations":[{"raw_affiliation_string":"TU Wien, Vienna, Austria","institution_ids":["https://openalex.org/I145847075"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.70588235,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"4163","last_page":"4168"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10315","display_name":"Decision-Making and Behavioral Economics","score":0.041200000792741776,"subfield":{"id":"https://openalex.org/subfields/1800","display_name":"General Decision Sciences"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10315","display_name":"Decision-Making and Behavioral Economics","score":0.041200000792741776,"subfield":{"id":"https://openalex.org/subfields/1800","display_name":"General Decision Sciences"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11118","display_name":"Evolutionary Psychology and Human Behavior","score":0.021400000900030136,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10081","display_name":"Auditing, Earnings Management, Governance","score":0.01940000057220459,"subfield":{"id":"https://openalex.org/subfields/1402","display_name":"Accounting"},"field":{"id":"https://openalex.org/fields/14","display_name":"Business, Management and Accounting"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/contamination","display_name":"Contamination","score":0.670199990272522},{"id":"https://openalex.org/keywords/biocide","display_name":"Biocide","score":0.2257000058889389},{"id":"https://openalex.org/keywords/vomitoxin","display_name":"Vomitoxin","score":0.2223999947309494},{"id":"https://openalex.org/keywords/component","display_name":"Component (thermodynamics)","score":0.22210000455379486}],"concepts":[{"id":"https://openalex.org/C112570922","wikidata":"https://www.wikidata.org/wiki/Q60528603","display_name":"Contamination","level":2,"score":0.670199990272522},{"id":"https://openalex.org/C39432304","wikidata":"https://www.wikidata.org/wiki/Q188847","display_name":"Environmental science","level":0,"score":0.3993000090122223},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.39579999446868896},{"id":"https://openalex.org/C107872376","wikidata":"https://www.wikidata.org/wiki/Q321355","display_name":"Environmental chemistry","level":1,"score":0.2842000126838684},{"id":"https://openalex.org/C548081761","wikidata":"https://www.wikidata.org/wiki/Q180388","display_name":"Waste management","level":1,"score":0.2736000120639801},{"id":"https://openalex.org/C197471484","wikidata":"https://www.wikidata.org/wiki/Q864939","display_name":"Biocide","level":2,"score":0.2257000058889389},{"id":"https://openalex.org/C2779741023","wikidata":"https://www.wikidata.org/wiki/Q420518","display_name":"Vomitoxin","level":4,"score":0.2223999947309494},{"id":"https://openalex.org/C168167062","wikidata":"https://www.wikidata.org/wiki/Q1117970","display_name":"Component (thermodynamics)","level":2,"score":0.22210000455379486},{"id":"https://openalex.org/C134121241","wikidata":"https://www.wikidata.org/wiki/Q899301","display_name":"Yield (engineering)","level":2,"score":0.20430000126361847},{"id":"https://openalex.org/C93497512","wikidata":"https://www.wikidata.org/wiki/Q5164919","display_name":"Contamination control","level":3,"score":0.19679999351501465}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3805712.3809901","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3805712.3809901","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 49th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/2","score":0.4068903923034668,"display_name":"Zero hunger"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":16,"referenced_works":["https://openalex.org/W2006996936","https://openalex.org/W2022871093","https://openalex.org/W2307814545","https://openalex.org/W2410091547","https://openalex.org/W3100107515","https://openalex.org/W3136109765","https://openalex.org/W4288280763","https://openalex.org/W4312980078","https://openalex.org/W4384891026","https://openalex.org/W4404355224","https://openalex.org/W4409159862","https://openalex.org/W4412376999","https://openalex.org/W4412396188","https://openalex.org/W4412930207","https://openalex.org/W4416018066","https://openalex.org/W7090765597"],"related_works":[],"abstract_inverted_index":{"Benchmark":[0],"collections":[1],"have":[2],"long":[3],"enabled":[4],"controlled":[5],"comparison":[6],"and":[7,66,113,136,164],"cumulative":[8],"progress":[9],"in":[10,26,41,193],"Information":[11],"Retrieval":[12,74],"(IR).":[13],"However,":[14,123],"prior":[15],"meta-analyses":[16],"show":[17],"that":[18],"reported":[19],"effectiveness":[20,159],"gains":[21],"often":[22],"fail":[23],"to":[24,29,130,171],"accumulate,":[25],"part":[27],"due":[28],"weak":[30],"or":[31,204],"outdated":[32],"baselines.":[33],"Large":[34],"language":[35],"models":[36],"(LLMs)":[37],"are":[38],"increasingly":[39],"used":[40],"retrieval":[42],"pipelines,":[43],"yet":[44],"their":[45],"impact":[46],"on":[47,61,105,116],"established":[48],"IR":[49],"benchmarks":[50],"has":[51,126],"not":[52,198],"been":[53],"systematically":[54],"analyzed.":[55],"We":[56,88,183],"analyze":[57],"179":[58],"publications":[59],"reporting":[60],"the":[62,67,108,119,133,140,165,185,194],"TREC":[63,68,110],"Robust04":[64,117],"collection":[65],"Deep":[69],"Learning":[70],"2020":[71,111],"(DL20)":[72],"Passage":[73],"benchmark,":[75],"using":[76],"ACM":[77],"Digital":[78],"Library":[79],"keyword":[80],"search":[81],"supplemented":[82],"by":[83],"citation-graph":[84],"backtracking":[85],"for":[86],"Robust04.":[87],"observe":[89],"what":[90],"we":[91],"term":[92],"an":[93],"LLM":[94,99,150,186],"effect:":[95],"recent":[96],"systems":[97],"incorporating":[98],"components":[100],"achieve":[101],"8.8%":[102],"higher":[103,115],"nDCG@10":[104,131],"DL20":[106],"than":[107,118],"best":[109],"result":[112],"11.9%":[114],"strongest":[120],"pre-2024":[121],"result.":[122],"evaluation":[124],"practice":[125],"shifted":[127],"from":[128,175,201],"MAP":[129],"over":[132],"same":[134],"window,":[135],"our":[137],"adaptation":[138],"of":[139,167],"Data":[141],"Contamination":[142],"Quiz":[143],"reveals":[144],"12-41%":[145],"contamination":[146,169],"across":[147],"two":[148],"widely-used":[149],"rerankers.":[151],"Filtering":[152],"contaminated":[153],"topics":[154],"shows":[155],"no":[156],"statistically":[157],"significant":[158],"difference,":[160],"but":[161,190,197],"small":[162],"samples":[163],"uncertainty":[166],"adapting":[168],"detection":[170],"reranking":[172],"prevent":[173],"us":[174],"ruling":[176],"out":[177],"memorization":[178],"as":[179,188],"a":[180],"contributing":[181],"factor.":[182],"read":[184],"effect":[187],"real":[189],"unverified:":[191],"visible":[192],"aggregate":[195],"numbers,":[196],"cleanly":[199],"separable":[200],"metric":[202],"drift":[203],"pretraining":[205],"overlap.":[206]},"counts_by_year":[],"updated_date":"2026-07-17T05:52:16.776730","created_date":"2026-07-16T00:00:00"}