{"id":"https://openalex.org/W4387725656","doi":"https://doi.org/10.48550/arxiv.2310.10628","title":"Data Contamination Through the Lens of Time","display_name":"Data Contamination Through the Lens of Time","publication_year":2023,"publication_date":"2023-10-16","ids":{"openalex":"https://openalex.org/W4387725656","doi":"https://doi.org/10.48550/arxiv.2310.10628"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2310.10628","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2310.10628","pdf_url":"https://arxiv.org/pdf/2310.10628","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2310.10628","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5041830853","display_name":"Manley Roberts","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Roberts, Manley","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5014196569","display_name":"Himanshu Thakur","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Thakur, Himanshu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5037557476","display_name":"Christine P. Herlihy","orcid":"https://orcid.org/0000-0001-7405-603X","observed_orcids":["https://orcid.org/0000-0001-7405-603X"]},"institutions":[{"id":"https://openalex.org/I66946132","display_name":"University of Maryland, College Park","ror":"https://ror.org/047s2c258","country_code":"US","type":"education","lineage":["https://openalex.org/I66946132"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Herlihy, Christine","raw_affiliation_strings":["University of Maryland ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Maryland ,","institution_ids":["https://openalex.org/I66946132"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5113475873","display_name":"Colin White","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I122411786","display_name":"California Institute of Technology","ror":"https://ror.org/05dxps055","country_code":"US","type":"education","lineage":["https://openalex.org/I122411786"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"White, Colin","raw_affiliation_strings":["Caltech"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Caltech","institution_ids":["https://openalex.org/I122411786"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5063355813","display_name":"Samuel W. Dooley","orcid":"https://orcid.org/0000-0001-8598-7681","observed_orcids":["https://orcid.org/0000-0001-8598-7681"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Dooley, Samuel","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":47,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.6660000085830688,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.6660000085830688,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.16169999539852142,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.04859999939799309,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/data-contamination","display_name":"contaminated data","score":0.9940000176429749},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.9710000157356262},{"id":"https://openalex.org/keywords/benchmark-evaluation","display_name":"benchmark evaluation","score":0.7570000290870667},{"id":"https://openalex.org/keywords/gpt","display_name":"GPT","score":0.6909999847412109},{"id":"https://openalex.org/keywords/codeforces","display_name":"Codeforces","score":0.8370000123977661},{"id":"https://openalex.org/keywords/mathematical-problem-solving","display_name":"mathematical problem solving","score":0.8840000033378601},{"id":"https://openalex.org/keywords/longitudinal-analysis","display_name":"longitudinal analysis","score":0.6430000066757202}],"concepts":[{"id":"https://openalex.org/C112570922","wikidata":"https://www.wikidata.org/wiki/Q60528603","display_name":"Contamination","level":2,"score":0.5938761234283447},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5929824113845825},{"id":"https://openalex.org/C132964779","wikidata":"https://www.wikidata.org/wiki/Q2110223","display_name":"Raw data","level":2,"score":0.5222883224487305},{"id":"https://openalex.org/C2780586970","wikidata":"https://www.wikidata.org/wiki/Q1357284","display_name":"Popularity","level":2,"score":0.5192182660102844},{"id":"https://openalex.org/C43091099","wikidata":"https://www.wikidata.org/wiki/Q1067788","display_name":"Through-the-lens metering","level":3,"score":0.5000104904174805},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.4379497766494751},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.42321568727493286},{"id":"https://openalex.org/C15336307","wikidata":"https://www.wikidata.org/wiki/Q1766051","display_name":"Lens (geology)","level":2,"score":0.2882479429244995},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.19288402795791626},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.159121572971344},{"id":"https://openalex.org/C18903297","wikidata":"https://www.wikidata.org/wiki/Q7150","display_name":"Ecology","level":1,"score":0.10598757863044739},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.08310860395431519},{"id":"https://openalex.org/C78762247","wikidata":"https://www.wikidata.org/wiki/Q1273174","display_name":"Petroleum engineering","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C78519656","wikidata":"https://www.wikidata.org/wiki/Q101333","display_name":"Mechanical engineering","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2310.10628","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2310.10628","pdf_url":"https://arxiv.org/pdf/2310.10628","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2310.10628","doi":"https://doi.org/10.48550/arxiv.2310.10628","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2310.10628","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2310.10628","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2310.10628","pdf_url":"https://arxiv.org/pdf/2310.10628","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4387725656.pdf","grobid_xml":"https://content.openalex.org/works/W4387725656.grobid-xml"},"referenced_works_count":14,"referenced_works":["https://openalex.org/W2953320089","https://openalex.org/W3213368902","https://openalex.org/W4224308101","https://openalex.org/W4226287673","https://openalex.org/W4285600327","https://openalex.org/W4309395035","https://openalex.org/W4318149317","https://openalex.org/W4376167329","https://openalex.org/W4382173247","https://openalex.org/W4385565416","https://openalex.org/W4386081878","https://openalex.org/W4386185625","https://openalex.org/W4389518805","https://openalex.org/W7133260402"],"related_works":["https://openalex.org/W4404783013","https://openalex.org/W4407806804","https://openalex.org/W6939251796","https://openalex.org/W4402670101","https://openalex.org/W4416346372","https://openalex.org/W4388555916","https://openalex.org/W7166876690","https://openalex.org/W4390833695","https://openalex.org/W7027667564","https://openalex.org/W4385965989"],"abstract_inverted_index":{"Recent":[0],"claims":[1],"about":[2],"the":[3,26,46,78,90,150,177],"impressive":[4],"abilities":[5],"of":[6,25,32,65,83,93,136,155,166,179],"large":[7],"language":[8],"models":[9,98],"(LLMs)":[10],"are":[11,40],"often":[12],"supported":[13],"by":[14,88],"evaluating":[15,36],"publicly":[16,173],"available":[17],"benchmarks.":[18],"Since":[19],"LLMs":[20,87,180],"train":[21,182],"on":[22,37,183],"wide":[23],"swaths":[24],"internet,":[27],"this":[28,74],"practice":[29],"raises":[30],"concerns":[31],"data":[33,84,156],"contamination,":[34],"i.e.,":[35],"examples":[38],"that":[39,132,181],"explicitly":[41],"or":[42,70],"implicitly":[43],"included":[44],"in":[45,86,96,158,176],"training":[47,66,94],"data.":[48,185],"Data":[49],"contamination":[50,85,157],"remains":[51],"notoriously":[52],"challenging":[53],"to":[54,99],"measure":[55],"and":[56,114,117,129,144,169],"mitigate,":[57],"even":[58],"with":[59,163],"partial":[60],"attempts":[61],"like":[62],"controlled":[63],"experimentation":[64],"data,":[67],"canary":[68],"strings,":[69],"embedding":[71],"similarities.":[72],"In":[73],"work,":[75],"we":[76,107],"conduct":[77],"first":[79],"thorough":[80],"longitudinal":[81],"analysis":[82],"using":[89],"natural":[91],"experiment":[92],"cutoffs":[95],"GPT":[97],"look":[100],"at":[101],"benchmarks":[102,175],"released":[103],"over":[104],"time.":[105],"Specifically,":[106],"consider":[108],"two":[109],"code/mathematical":[110],"problem-solving":[111],"datasets,":[112],"Codeforces":[113],"Project":[115],"Euler,":[116],"find":[118],"statistically":[119],"significant":[120],"trends":[121],"among":[122],"LLM":[123],"pass":[124],"rate":[125],"vs.":[126],"GitHub":[127],"popularity":[128],"release":[130],"date":[131],"provide":[133],"strong":[134],"evidence":[135],"contamination.":[137],"By":[138],"open-sourcing":[139],"our":[140,147],"dataset,":[141],"raw":[142],"results,":[143],"evaluation":[145],"framework,":[146],"work":[148],"paves":[149],"way":[151],"for":[152,172],"rigorous":[153],"analyses":[154],"modern":[159],"models.":[160],"We":[161],"conclude":[162],"a":[164],"discussion":[165],"best":[167],"practices":[168],"future":[170],"steps":[171],"releasing":[174],"age":[178],"webscale":[184]},"counts_by_year":[{"year":2026,"cited_by_count":11},{"year":2025,"cited_by_count":14},{"year":2024,"cited_by_count":22}],"updated_date":"2026-11-11T12:16:26.678958","created_date":"2023-10-18T00:00:00"}