{"id":"https://openalex.org/W4404783013","doi":"https://doi.org/10.18653/v1/2024.findings-emnlp.30","title":"An Open-Source Data Contamination Report for Large Language Models","display_name":"An Open-Source Data Contamination Report for Large Language Models","publication_year":2024,"publication_date":"2024-01-01","ids":{"openalex":"https://openalex.org/W4404783013","doi":"https://doi.org/10.18653/v1/2024.findings-emnlp.30"},"language":"en","primary_location":{"id":"doi:10.18653/v1/2024.findings-emnlp.30","doi":"https://doi.org/10.18653/v1/2024.findings-emnlp.30","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2024.findings-emnlp.30","pdf_url":"https://aclanthology.org/2024.findings-emnlp.30.pdf","source":{"id":"https://openalex.org/S7407096085","display_name":"Findings of the Association for Computational Linguistics: EMNLP (Findings of EMNLP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: EMNLP 2024","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://aclanthology.org/2024.findings-emnlp.30.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100381401","display_name":"Yucheng Li","orcid":"https://orcid.org/0000-0003-3473-3176","observed_orcids":["https://orcid.org/0000-0003-3473-3176"]},"institutions":[{"id":"https://openalex.org/I28290843","display_name":"University of Surrey","ror":"https://ror.org/00ks66431","country_code":"GB","type":"education","lineage":["https://openalex.org/I28290843"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Yucheng Li","raw_affiliation_strings":["University of Surrey"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Surrey","institution_ids":["https://openalex.org/I28290843"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5000688953","display_name":"Yunhao Guo","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I151727225","display_name":"Harbin Engineering University","ror":"https://ror.org/03x80pn82","country_code":"CN","type":"education","lineage":["https://openalex.org/I151727225"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yunhao Guo","raw_affiliation_strings":["Harbin Engineering University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Harbin Engineering University","institution_ids":["https://openalex.org/I151727225"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5090217029","display_name":"Frank Gu\u00e9rin","orcid":"https://orcid.org/0000-0003-1918-6311","observed_orcids":["https://orcid.org/0000-0003-1918-6311"]},"institutions":[{"id":"https://openalex.org/I28290843","display_name":"University of Surrey","ror":"https://ror.org/00ks66431","country_code":"GB","type":"education","lineage":["https://openalex.org/I28290843"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Frank Guerin","raw_affiliation_strings":["University of Surrey"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Surrey","institution_ids":["https://openalex.org/I28290843"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5024599321","display_name":"Chenghua Lin","orcid":"https://orcid.org/0000-0003-3454-2468","observed_orcids":["https://orcid.org/0000-0003-3454-2468"]},"institutions":[{"id":"https://openalex.org/I28407311","display_name":"University of Manchester","ror":"https://ror.org/027m9bs27","country_code":"GB","type":"education","lineage":["https://openalex.org/I28407311"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Chenghua Lin","raw_affiliation_strings":["University of Manchester"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Manchester","institution_ids":["https://openalex.org/I28407311"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":8.943,"has_fulltext":true,"cited_by_count":37,"citation_normalized_percentile":{"value":0.98607598,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":99,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"528","last_page":"541"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.8618000149726868,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.8618000149726868,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.06019999831914902,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.049800001084804535,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/data-contamination","display_name":"contaminated data","score":0.9940000176429749},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.9769999980926514},{"id":"https://openalex.org/keywords/model-evaluation","display_name":"model evaluation","score":0.6539999842643738},{"id":"https://openalex.org/keywords/memorization","display_name":"memorization","score":0.7170000076293945},{"id":"https://openalex.org/keywords/benchmark-contamination","display_name":"benchmark contamination","score":0.3540000021457672},{"id":"https://openalex.org/keywords/multiple-choice-question-answering","display_name":"multiple-choice question answering","score":0.7910000085830688},{"id":"https://openalex.org/keywords/hellaswag","display_name":"HellaSwag","score":0.9700000286102295},{"id":"https://openalex.org/keywords/mmlu","display_name":"MMLU","score":0.9990000128746033},{"id":"https://openalex.org/keywords/transparency-in-assessment","display_name":"assessment transparency","score":0.3370000123977661}],"concepts":[{"id":"https://openalex.org/C112570922","wikidata":"https://www.wikidata.org/wiki/Q60528603","display_name":"Contamination","level":2,"score":0.6844683885574341},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6661770343780518},{"id":"https://openalex.org/C3018397939","wikidata":"https://www.wikidata.org/wiki/Q3644502","display_name":"Open source","level":3,"score":0.6176612973213196},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.346943736076355},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.2558237910270691},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.12563011050224304},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C18903297","wikidata":"https://www.wikidata.org/wiki/Q7150","display_name":"Ecology","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.18653/v1/2024.findings-emnlp.30","doi":"https://doi.org/10.18653/v1/2024.findings-emnlp.30","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2024.findings-emnlp.30","pdf_url":"https://aclanthology.org/2024.findings-emnlp.30.pdf","source":{"id":"https://openalex.org/S7407096085","display_name":"Findings of the Association for Computational Linguistics: EMNLP (Findings of EMNLP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: EMNLP 2024","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2310.17589","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2310.17589","pdf_url":"https://arxiv.org/pdf/2310.17589","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2310.17589","doi":"https://doi.org/10.48550/arxiv.2310.17589","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2310.17589","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.18653/v1/2024.findings-emnlp.30","doi":"https://doi.org/10.18653/v1/2024.findings-emnlp.30","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2024.findings-emnlp.30","pdf_url":"https://aclanthology.org/2024.findings-emnlp.30.pdf","source":{"id":"https://openalex.org/S7407096085","display_name":"Findings of the Association for Computational Linguistics: EMNLP (Findings of EMNLP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: EMNLP 2024","raw_type":"proceedings-article"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4404783013.pdf","grobid_xml":"https://content.openalex.org/works/W4404783013.grobid-xml"},"referenced_works_count":32,"referenced_works":["https://openalex.org/W2898695519","https://openalex.org/W2936262349","https://openalex.org/W2950700477","https://openalex.org/W2951831170","https://openalex.org/W2953271402","https://openalex.org/W2963323070","https://openalex.org/W3195140297","https://openalex.org/W3198002980","https://openalex.org/W3204096139","https://openalex.org/W4221159672","https://openalex.org/W4322718191","https://openalex.org/W4376653782","https://openalex.org/W4384918448","https://openalex.org/W4385571641","https://openalex.org/W4385572388","https://openalex.org/W4385965989","https://openalex.org/W4386576726","https://openalex.org/W4386908094","https://openalex.org/W4387427395","https://openalex.org/W4387725656","https://openalex.org/W4387985331","https://openalex.org/W4387994765","https://openalex.org/W4388185104","https://openalex.org/W4391376689","https://openalex.org/W4391506345","https://openalex.org/W4391583807","https://openalex.org/W4391591446","https://openalex.org/W4400141990","https://openalex.org/W4401042664","https://openalex.org/W4401042689","https://openalex.org/W4402671052","https://openalex.org/W4409181908"],"related_works":["https://openalex.org/W4407806804","https://openalex.org/W4402670101","https://openalex.org/W4387725656","https://openalex.org/W4403679014","https://openalex.org/W4404370628","https://openalex.org/W4404088284","https://openalex.org/W4416035762","https://openalex.org/W4404782840","https://openalex.org/W7027667564","https://openalex.org/W4389518953"],"abstract_inverted_index":{"Data":[0],"contamination":[1,28,43,71,80,99,110,124,161],"in":[2,72],"model":[3,37,52,130],"evaluation":[4,38],"has":[5,30,162],"become":[6,31],"increasingly":[7],"prevalent":[8],"with":[9,108],"the":[10,109],"growing":[11],"popularity":[12],"of":[13,24,35,117,134],"large":[14,50,86,118],"language":[15,51,87,119],"models.It":[16],"allows":[17],"models":[18,88,120,172],"to":[19,39,67,104,136,167],"\"cheat\"":[20],"via":[21],"memorisation":[22],"instead":[23],"displaying":[25],"true":[26],"capabilities.Therefore,":[27],"analysis":[29,44,116],"a":[32,149],"crucial":[33],"part":[34],"reliable":[36],"validate":[40],"results.However,":[41],"existing":[42],"is":[45,152],"usually":[46],"conducted":[47],"internally":[48],"by":[49],"developers":[53],"and":[54,58,64,138,145,148,169],"often":[55],"lacks":[56],"transparency":[57],"completeness.This":[59],"paper":[60],"introduces":[61],"an":[62,77],"efficient":[63],"affordable":[65],"method":[66],"identify":[68],"potential":[69],"data":[70,79,123,160],"LLM":[73],"benchmarks.We":[74],"also":[75,157],"present":[76],"extensive":[78],"report":[81],"for":[82],"over":[83,114],"15":[84],"popular":[85],"across":[89,106],"six":[90],"widely":[91],"used":[92],"multiple-choice":[93],"QA":[94],"benchmarks.Our":[95],"experiments":[96],"reveal":[97],"varying":[98],"levels":[100],"ranging":[101],"from":[102,165,175],"1%":[103],"45%":[105],"benchmarks,":[107,147],"degree":[111],"increasing":[112],"rapidly":[113,164],"time.Performance":[115],"indicates":[121],"that":[122,159,170],"can":[125],"have":[126],"significant":[127],"impact":[128],"on":[129,142,154],"metrics:":[131],"inflated":[132],"accuracy":[133],"up":[135],"14%":[137],"7%":[139],"are":[140],"observed":[141],"contaminated":[143,155,176],"C-Eval":[144],"HellaSwag":[146],"small":[150],"increase":[151],"identified":[153],"MMLU.We":[156],"find":[158],"grown":[163],"2020":[166],"2023":[168],"larger":[171],"benefit":[173],"more":[174],"test":[177],"sets.":[178]},"counts_by_year":[{"year":2026,"cited_by_count":16},{"year":2025,"cited_by_count":10},{"year":2024,"cited_by_count":11}],"updated_date":"2026-11-04T10:54:05.112534","created_date":"2024-11-28T00:00:00"}