{"id":"https://openalex.org/W4402673633","doi":"https://doi.org/10.1109/e-science62913.2024.10678657","title":"Mission Reproducibility: An Investigation on Reproducibility Issues in Machine Learning and Information Retrieval Research","display_name":"Mission Reproducibility: An Investigation on Reproducibility Issues in Machine Learning and Information Retrieval Research","publication_year":2024,"publication_date":"2024-09-16","ids":{"openalex":"https://openalex.org/W4402673633","doi":"https://doi.org/10.1109/e-science62913.2024.10678657"},"language":"en","primary_location":{"id":"doi:10.1109/e-science62913.2024.10678657","is_oa":false,"landing_page_url":"https://doi.org/10.1109/e-science62913.2024.10678657","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2024 IEEE 20th International Conference on e-Science (e-Science)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5071125448","display_name":"Moritz Staudinger","orcid":"https://orcid.org/0000-0002-5164-2690"},"institutions":[{"id":"https://openalex.org/I145847075","display_name":"TU Wien","ror":"https://ror.org/04d836q62","country_code":"AT","type":"education","lineage":["https://openalex.org/I145847075"]}],"countries":["AT"],"is_corresponding":true,"raw_author_name":"Moritz Staudinger","raw_affiliation_strings":["TU Wien,Data Science,Vienna,Austria"],"affiliations":[{"raw_affiliation_string":"TU Wien,Data Science,Vienna,Austria","institution_ids":["https://openalex.org/I145847075"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5031589314","display_name":"Bettina Kern","orcid":"https://orcid.org/0000-0003-1591-7236"},"institutions":[{"id":"https://openalex.org/I145847075","display_name":"TU Wien","ror":"https://ror.org/04d836q62","country_code":"AT","type":"education","lineage":["https://openalex.org/I145847075"]}],"countries":["AT"],"is_corresponding":false,"raw_author_name":"Bettina M. J. Kern","raw_affiliation_strings":["TU Wien,Data Science,Vienna,Austria"],"affiliations":[{"raw_affiliation_string":"TU Wien,Data Science,Vienna,Austria","institution_ids":["https://openalex.org/I145847075"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5015351144","display_name":"Tomasz Miksa","orcid":"https://orcid.org/0000-0002-4929-7875"},"institutions":[{"id":"https://openalex.org/I145847075","display_name":"TU Wien","ror":"https://ror.org/04d836q62","country_code":"AT","type":"education","lineage":["https://openalex.org/I145847075"]}],"countries":["AT"],"is_corresponding":false,"raw_author_name":"Tomasz Miksa","raw_affiliation_strings":["TU Wien,Research Data Management,Vienna,Austria"],"affiliations":[{"raw_affiliation_string":"TU Wien,Research Data Management,Vienna,Austria","institution_ids":["https://openalex.org/I145847075"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5107607301","display_name":"Lukas Arnhold","orcid":"https://orcid.org/0009-0007-3410-5491"},"institutions":[{"id":"https://openalex.org/I145847075","display_name":"TU Wien","ror":"https://ror.org/04d836q62","country_code":"AT","type":"education","lineage":["https://openalex.org/I145847075"]}],"countries":["AT"],"is_corresponding":false,"raw_author_name":"Lukas Arnhold","raw_affiliation_strings":["TU Wien,Research Data Management,Vienna,Austria"],"affiliations":[{"raw_affiliation_string":"TU Wien,Research Data Management,Vienna,Austria","institution_ids":["https://openalex.org/I145847075"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5079362815","display_name":"Peter Knees","orcid":"https://orcid.org/0000-0003-3906-1292"},"institutions":[{"id":"https://openalex.org/I145847075","display_name":"TU Wien","ror":"https://ror.org/04d836q62","country_code":"AT","type":"education","lineage":["https://openalex.org/I145847075"]}],"countries":["AT"],"is_corresponding":false,"raw_author_name":"Peter Knees","raw_affiliation_strings":["TU Wien,Data Science,Vienna,Austria"],"affiliations":[{"raw_affiliation_string":"TU Wien,Data Science,Vienna,Austria","institution_ids":["https://openalex.org/I145847075"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5057690103","display_name":"Andreas Rauber","orcid":"https://orcid.org/0000-0002-9272-6225"},"institutions":[{"id":"https://openalex.org/I145847075","display_name":"TU Wien","ror":"https://ror.org/04d836q62","country_code":"AT","type":"education","lineage":["https://openalex.org/I145847075"]}],"countries":["AT"],"is_corresponding":false,"raw_author_name":"Andreas Rauber","raw_affiliation_strings":["TU Wien,Data Science,Vienna,Austria"],"affiliations":[{"raw_affiliation_string":"TU Wien,Data Science,Vienna,Austria","institution_ids":["https://openalex.org/I145847075"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5020665735","display_name":"Allan Hanbury","orcid":"https://orcid.org/0000-0002-7149-5843"},"institutions":[{"id":"https://openalex.org/I145847075","display_name":"TU Wien","ror":"https://ror.org/04d836q62","country_code":"AT","type":"education","lineage":["https://openalex.org/I145847075"]}],"countries":["AT"],"is_corresponding":false,"raw_author_name":"Allan Hanbury","raw_affiliation_strings":["TU Wien,Data Science,Vienna,Austria"],"affiliations":[{"raw_affiliation_string":"TU Wien,Data Science,Vienna,Austria","institution_ids":["https://openalex.org/I145847075"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":7,"corresponding_author_ids":["https://openalex.org/A5071125448"],"corresponding_institution_ids":["https://openalex.org/I145847075"],"apc_list":null,"apc_paid":null,"fwci":1.1085,"has_fulltext":false,"cited_by_count":2,"citation_normalized_percentile":{"value":0.83470685,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":95,"max":96},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"9"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11891","display_name":"Big Data and Business Intelligence","score":0.9818000197410583,"subfield":{"id":"https://openalex.org/subfields/1404","display_name":"Management Information Systems"},"field":{"id":"https://openalex.org/fields/14","display_name":"Business, Management and Accounting"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T11891","display_name":"Big Data and Business Intelligence","score":0.9818000197410583,"subfield":{"id":"https://openalex.org/subfields/1404","display_name":"Management Information Systems"},"field":{"id":"https://openalex.org/fields/14","display_name":"Business, Management and Accounting"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11719","display_name":"Data Quality and Management","score":0.9592999815940857,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/reproducibility","display_name":"Reproducibility","score":0.9551054239273071},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6152856349945068},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.3898182511329651},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3654644191265106},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.09950610995292664},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.07832342386245728}],"concepts":[{"id":"https://openalex.org/C9893847","wikidata":"https://www.wikidata.org/wiki/Q1425625","display_name":"Reproducibility","level":2,"score":0.9551054239273071},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6152856349945068},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.3898182511329651},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3654644191265106},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.09950610995292664},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.07832342386245728}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/e-science62913.2024.10678657","is_oa":false,"landing_page_url":"https://doi.org/10.1109/e-science62913.2024.10678657","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2024 IEEE 20th International Conference on e-Science (e-Science)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Gender equality","score":0.4099999964237213,"id":"https://metadata.un.org/sdg/5"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":36,"referenced_works":["https://openalex.org/W2811804","https://openalex.org/W311652463","https://openalex.org/W2302501749","https://openalex.org/W2307814545","https://openalex.org/W2443960221","https://openalex.org/W2508716390","https://openalex.org/W2704480242","https://openalex.org/W2790725087","https://openalex.org/W2799048248","https://openalex.org/W2962797668","https://openalex.org/W2964079897","https://openalex.org/W2969044820","https://openalex.org/W2987812053","https://openalex.org/W2989395196","https://openalex.org/W3016194403","https://openalex.org/W3016244823","https://openalex.org/W3017158675","https://openalex.org/W3149807132","https://openalex.org/W3152269143","https://openalex.org/W4284698816","https://openalex.org/W4312407404","https://openalex.org/W4378945316","https://openalex.org/W4384643624","https://openalex.org/W4386544286","https://openalex.org/W6726092351","https://openalex.org/W6732137801","https://openalex.org/W6750222608","https://openalex.org/W6763825385","https://openalex.org/W6764218318","https://openalex.org/W6775031676","https://openalex.org/W6784200677","https://openalex.org/W6792383182","https://openalex.org/W6793757958","https://openalex.org/W6802898477","https://openalex.org/W6966986966","https://openalex.org/W6996244326"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2748952813","https://openalex.org/W2413717610","https://openalex.org/W1973270181","https://openalex.org/W2417696084","https://openalex.org/W2368782778","https://openalex.org/W2087830269","https://openalex.org/W3106281778","https://openalex.org/W3134060498","https://openalex.org/W2003643616"],"abstract_inverted_index":{"This":[0],"paper":[1],"analyzes":[2],"the":[3,22,29,56,64,91,124,132,154],"most":[4],"common":[5],"problems":[6,71],"limiting":[7],"reproducibility":[8,23],"of":[9,24,31,100],"Information":[10],"Retrieval":[11],"research":[12,176],"and":[13,18,26,59,61,78,81,104,163,179],"provides":[14],"researchers":[15],"with":[16,85],"insights":[17,66],"guidelines":[19],"to":[20,27,87,116,123,138,172],"improve":[21],"experiments":[25],"allow":[28],"verification":[30],"obtained":[32],"results.":[33],"We":[34,54],"conducted":[35],"a":[36,117],"study":[37],"on":[38,90,97],"45":[39],"reproduction":[40,92],"reports":[41,57],"off":[42],"17":[43,109],"different":[44,65,68,110,155],"papers,":[45,111,156],"which":[46],"have":[47],"been":[48],"published":[49],"at":[50,147],"renowned":[51],"IR":[52],"conferences.":[53],"analyzed":[55,84],"qualitatively":[58],"quantitatively":[60],"looked":[62],"into":[63,74],"from":[67],"groups.":[69],"Occurring":[70],"are":[72],"classified":[73],"three":[75],"problem":[76],"families":[77],"13":[79],"categories":[80],"afre":[82],"then":[83],"respect":[86],"their":[88,98],"influence":[89],"process":[93],"as":[94,96,157],"well":[95],"frequency":[99],"appearance":[101],"over":[102],"time":[103],"per":[105],"conference.":[106],"Of":[107],"these":[108],"14":[112],"papers":[113],"were":[114,150,161],"reproducible":[115,136,178],"certain":[118],"degree":[119],"without":[120],"significant":[121],"differences":[122],"original":[125],"results,":[126],"but":[127],"in":[128],"many":[129],"cases":[130],"not":[131],"whole":[133],"experiment":[134,159],"was":[135,165],"due":[137],"missing":[139],"code,":[140],"information":[141,164],"or":[142],"data.":[143],"Also,":[144],"we":[145,169],"look":[146],"assumptions":[148],"that":[149],"made":[151],"when":[152],"reproducing":[153],"some":[158],"workflows":[160],"incomplete":[162],"missing.":[166],"In":[167],"addition,":[168],"propose":[170],"recommendations":[171],"make":[173],"machine":[174],"learning":[175],"more":[177],"FAIR.":[180]},"counts_by_year":[{"year":2025,"cited_by_count":2}],"updated_date":"2025-12-21T01:58:51.020947","created_date":"2025-10-10T00:00:00"}