{"id":"https://openalex.org/W4403749567","doi":"https://doi.org/10.48550/arxiv.2407.12220","title":"Questionable practices in machine learning","display_name":"Questionable practices in machine learning","publication_year":2024,"publication_date":"2024-07-17","ids":{"openalex":"https://openalex.org/W4403749567","doi":"https://doi.org/10.48550/arxiv.2407.12220"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2407.12220","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2407.12220","pdf_url":"https://arxiv.org/pdf/2407.12220","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2407.12220","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5029915724","display_name":"Gavin Leech","orcid":"https://orcid.org/0000-0002-9298-1488","observed_orcids":["https://orcid.org/0000-0002-9298-1488"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Leech, Gavin","raw_affiliation_strings":["Arb Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Arb Research","institution_ids":[]}]},{"author_position":"middle","author":{"id":null,"display_name":"Vazquez, Juan J.","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Vazquez, Juan J.","raw_affiliation_strings":["Arb Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Arb Research","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5104631530","display_name":"Niclas K\u00fcpper","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I51601045","display_name":"University of Bath","ror":"https://ror.org/002h8g185","country_code":"GB","type":"education","lineage":["https://openalex.org/I51601045"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Kupper, Niclas","raw_affiliation_strings":["University of Bath"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Bath","institution_ids":["https://openalex.org/I51601045"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5056230521","display_name":"Misha Yagudin","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yagudin, Misha","raw_affiliation_strings":["Arb Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Arb Research","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5062539506","display_name":"Laurence Aitchison","orcid":"https://orcid.org/0000-0003-3681-4607","observed_orcids":["https://orcid.org/0000-0003-3681-4607"]},"institutions":[{"id":"https://openalex.org/I36234482","display_name":"University of Bristol","ror":"https://ror.org/0524sp257","country_code":"GB","type":"education","lineage":["https://openalex.org/I36234482"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Aitchison, Laurence","raw_affiliation_strings":["University of Bristol"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Bristol","institution_ids":["https://openalex.org/I36234482"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":21,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11492","display_name":"Academic integrity and plagiarism","score":0.8589000105857849,"subfield":{"id":"https://openalex.org/subfields/3304","display_name":"Education"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T11492","display_name":"Academic integrity and plagiarism","score":0.8589000105857849,"subfield":{"id":"https://openalex.org/subfields/3304","display_name":"Education"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.026799999177455902,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.01850000023841858,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/questionable-research-practices","display_name":"questionable research practices","score":0.9990000128746033},{"id":"https://openalex.org/keywords/machine-learning-model-evaluation","display_name":"machine learning model evaluation","score":0.8399999737739563},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.8999999761581421},{"id":"https://openalex.org/keywords/benchmark-evaluation","display_name":"benchmark evaluation","score":0.8309999704360962},{"id":"https://openalex.org/keywords/research-integrity","display_name":"research integrity","score":0.6060000061988831},{"id":"https://openalex.org/keywords/scientific-fraud","display_name":"scientific fraud","score":0.4429999887943268},{"id":"https://openalex.org/keywords/reproducibility","display_name":"reproducibility","score":0.515999972820282}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5080187320709229},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.37098532915115356}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2407.12220","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2407.12220","pdf_url":"https://arxiv.org/pdf/2407.12220","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2407.12220","doi":"https://doi.org/10.48550/arxiv.2407.12220","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2407.12220","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2407.12220","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2407.12220","pdf_url":"https://arxiv.org/pdf/2407.12220","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G49701852","display_name":null,"funder_award_id":"EP/S022937/1","funder_id":"https://openalex.org/F4320314731","funder_display_name":"UK Research and Innovation"}],"funders":[{"id":"https://openalex.org/F4320314731","display_name":"UK Research and Innovation","ror":"https://ror.org/001aqnf71"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4403749567.pdf","grobid_xml":"https://content.openalex.org/works/W4403749567.grobid-xml"},"referenced_works_count":88,"referenced_works":["https://openalex.org/W1555759181","https://openalex.org/W1821462560","https://openalex.org/W1988037271","https://openalex.org/W2072566913","https://openalex.org/W2594183968","https://openalex.org/W2739255396","https://openalex.org/W2799054028","https://openalex.org/W2881747041","https://openalex.org/W2898700502","https://openalex.org/W2910385168","https://openalex.org/W2912218663","https://openalex.org/W2938011096","https://openalex.org/W2944441143","https://openalex.org/W2950700477","https://openalex.org/W2971677963","https://openalex.org/W2987577970","https://openalex.org/W3005700362","https://openalex.org/W3083410900","https://openalex.org/W3092041776","https://openalex.org/W3134642945","https://openalex.org/W3146670357","https://openalex.org/W3156470785","https://openalex.org/W3157225713","https://openalex.org/W3174644206","https://openalex.org/W3177813494","https://openalex.org/W3199892937","https://openalex.org/W3210045394","https://openalex.org/W3210904070","https://openalex.org/W4220681836","https://openalex.org/W4221145126","https://openalex.org/W4224280420","https://openalex.org/W4285242720","https://openalex.org/W4287854593","https://openalex.org/W4295884246","https://openalex.org/W4313447114","https://openalex.org/W4320003957","https://openalex.org/W4321524373","https://openalex.org/W4327810158","https://openalex.org/W4367000491","https://openalex.org/W4378509498","https://openalex.org/W4379089709","https://openalex.org/W4381110793","https://openalex.org/W4381586770","https://openalex.org/W4384071683","https://openalex.org/W4384918448","https://openalex.org/W4384920109","https://openalex.org/W4385565080","https://openalex.org/W4385570463","https://openalex.org/W4385734111","https://openalex.org/W4385965989","https://openalex.org/W4387441491","https://openalex.org/W4387561528","https://openalex.org/W4387799916","https://openalex.org/W4387985331","https://openalex.org/W4387994765","https://openalex.org/W4388555916","https://openalex.org/W4388926373","https://openalex.org/W4389156617","https://openalex.org/W4389364446","https://openalex.org/W4389984066","https://openalex.org/W4390041933","https://openalex.org/W4390528013","https://openalex.org/W4390723197","https://openalex.org/W4390833695","https://openalex.org/W4391670970","https://openalex.org/W4392427259","https://openalex.org/W4392679398","https://openalex.org/W4392681182","https://openalex.org/W4392781833","https://openalex.org/W4394907038","https://openalex.org/W4395474395","https://openalex.org/W4399115929","https://openalex.org/W4399362671","https://openalex.org/W4399455140","https://openalex.org/W4400222341","https://openalex.org/W4400337927","https://openalex.org/W4400601293","https://openalex.org/W4400611194","https://openalex.org/W4400739958","https://openalex.org/W4401208776","https://openalex.org/W4401951602","https://openalex.org/W4402671023","https://openalex.org/W4406492615","https://openalex.org/W4409262649","https://openalex.org/W4409348063","https://openalex.org/W4415796423","https://openalex.org/W4415797772","https://openalex.org/W6941243218"],"related_works":["https://openalex.org/W7214033057","https://openalex.org/W7154706802","https://openalex.org/W4407124756","https://openalex.org/W4409415146","https://openalex.org/W4412888347","https://openalex.org/W7214196382","https://openalex.org/W4385018165","https://openalex.org/W6969433231","https://openalex.org/W7171749523","https://openalex.org/W7219103540"],"abstract_inverted_index":{"Evaluating":[0],"modern":[1],"ML":[2],"models":[3,59],"is":[4],"hard.":[5],"The":[6],"strong":[7],"incentive":[8],"for":[9,78],"researchers":[10,80],"and":[11],"companies":[12],"to":[13,23,81],"report":[14],"a":[15],"state-of-the-art":[16],"result":[17],"on":[18,61,84],"some":[19],"metric":[20],"often":[21],"leads":[22],"questionable":[24],"research":[25,35,68],"practices":[26,29,41],"(QRPs):":[27],"bad":[28],"which":[30,42],"fall":[31],"short":[32],"of":[33,56],"outright":[34],"fraud.":[36],"We":[37,64],"describe":[38],"44":[39],"such":[40],"can":[43],"undermine":[44],"reported":[45],"results,":[46],"giving":[47],"examples":[48],"where":[49],"possible.":[50],"Our":[51],"list":[52],"emphasises":[53],"the":[54],"evaluation":[55],"large":[57],"language":[58],"(LLMs)":[60],"public":[62],"benchmarks.":[63],"also":[65],"discuss":[66],"\"irreproducible":[67],"practices\",":[69],"i.e.":[70],"decisions":[71],"that":[72],"make":[73],"it":[74],"difficult":[75],"or":[76,85],"impossible":[77],"other":[79],"reproduce,":[82],"build":[83],"audit":[86],"previous":[87],"research.":[88]},"counts_by_year":[{"year":2026,"cited_by_count":10},{"year":2025,"cited_by_count":9},{"year":2024,"cited_by_count":2}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2024-10-25T00:00:00"}