{"id":"https://openalex.org/W4284697843","doi":"https://doi.org/10.1145/3477495.3532686","title":"Towards Reproducible Machine Learning Research in Information Retrieval","display_name":"Towards Reproducible Machine Learning Research in Information Retrieval","publication_year":2022,"publication_date":"2022-07-06","ids":{"openalex":"https://openalex.org/W4284697843","doi":"https://doi.org/10.1145/3477495.3532686"},"language":"en","primary_location":{"id":"doi:10.1145/3477495.3532686","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3477495.3532686","pdf_url":null,"source":{"id":"https://openalex.org/S4363608773","display_name":"Proceedings of the 45th International ACM SIGIR Conference on Research and Development in Information Retrieval","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 45th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5018957074","display_name":"Ana Lu\u010di\u0107","orcid":"https://orcid.org/0000-0002-5509-445X"},"institutions":[{"id":"https://openalex.org/I4210135670","display_name":"Amsterdam University of the Arts","ror":"https://ror.org/04dde1554","country_code":"NL","type":"education","lineage":["https://openalex.org/I4210135670"]},{"id":"https://openalex.org/I887064364","display_name":"University of Amsterdam","ror":"https://ror.org/04dkp9463","country_code":"NL","type":"education","lineage":["https://openalex.org/I887064364"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Ana Lucic","raw_affiliation_strings":["University of Amsterdam, Amsterdam, Netherlands"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Amsterdam, Amsterdam, Netherlands","institution_ids":["https://openalex.org/I4210135670","https://openalex.org/I887064364"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5056994081","display_name":"Maurits Bleeker","orcid":null},"institutions":[{"id":"https://openalex.org/I4210135670","display_name":"Amsterdam University of the Arts","ror":"https://ror.org/04dde1554","country_code":"NL","type":"education","lineage":["https://openalex.org/I4210135670"]},{"id":"https://openalex.org/I887064364","display_name":"University of Amsterdam","ror":"https://ror.org/04dkp9463","country_code":"NL","type":"education","lineage":["https://openalex.org/I887064364"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Maurits Bleeker","raw_affiliation_strings":["University of Amsterdam, Amsterdam, Netherlands"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Amsterdam, Amsterdam, Netherlands","institution_ids":["https://openalex.org/I4210135670","https://openalex.org/I887064364"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5031439294","display_name":"Maarten de Rijke","orcid":"https://orcid.org/0000-0002-1086-0202"},"institutions":[{"id":"https://openalex.org/I4210135670","display_name":"Amsterdam University of the Arts","ror":"https://ror.org/04dde1554","country_code":"NL","type":"education","lineage":["https://openalex.org/I4210135670"]},{"id":"https://openalex.org/I887064364","display_name":"University of Amsterdam","ror":"https://ror.org/04dkp9463","country_code":"NL","type":"education","lineage":["https://openalex.org/I887064364"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Maarten de Rijke","raw_affiliation_strings":["University of Amsterdam, Amsterdam, Netherlands"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Amsterdam, Amsterdam, Netherlands","institution_ids":["https://openalex.org/I4210135670","https://openalex.org/I887064364"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5109576489","display_name":"Koustuv Sinha","orcid":null},"institutions":[{"id":"https://openalex.org/I5023651","display_name":"McGill University","ror":"https://ror.org/01pxwe438","country_code":"CA","type":"education","lineage":["https://openalex.org/I5023651"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Koustuv Sinha","raw_affiliation_strings":["McGill University, Montreal, Canada"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"McGill University, Montreal, Canada","institution_ids":["https://openalex.org/I5023651"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5020170776","display_name":"Sami Jullien","orcid":"https://orcid.org/0000-0003-4507-6335"},"institutions":[{"id":"https://openalex.org/I4210135670","display_name":"Amsterdam University of the Arts","ror":"https://ror.org/04dde1554","country_code":"NL","type":"education","lineage":["https://openalex.org/I4210135670"]},{"id":"https://openalex.org/I887064364","display_name":"University of Amsterdam","ror":"https://ror.org/04dkp9463","country_code":"NL","type":"education","lineage":["https://openalex.org/I887064364"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Sami Jullien","raw_affiliation_strings":["University of Amsterdam, Amsterdam, Netherlands"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Amsterdam, Amsterdam, Netherlands","institution_ids":["https://openalex.org/I4210135670","https://openalex.org/I887064364"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5080857815","display_name":"Robert Stojnic","orcid":"https://orcid.org/0000-0002-6458-0273"},"institutions":[{"id":"https://openalex.org/I4210111288","display_name":"Meta (United Kingdom)","ror":"https://ror.org/020ye1821","country_code":"GB","type":"company","lineage":["https://openalex.org/I4210111288","https://openalex.org/I4210114444"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Robert Stojnic","raw_affiliation_strings":["Facebook AI Research, London, United Kingdom"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Facebook AI Research, London, United Kingdom","institution_ids":["https://openalex.org/I4210111288"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":6,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.2076,"has_fulltext":false,"cited_by_count":3,"citation_normalized_percentile":{"value":0.3948582,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"3459","last_page":"3461"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.9983000159263611,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.9983000159263611,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.9894999861717224,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.9815000295639038,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/reproducibility","display_name":"Reproducibility","score":0.8629893064498901},{"id":"https://openalex.org/keywords/checklist","display_name":"Checklist","score":0.7388284206390381},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6997463703155518},{"id":"https://openalex.org/keywords/artifact","display_name":"Artifact (error)","score":0.6199007630348206},{"id":"https://openalex.org/keywords/field","display_name":"Field (mathematics)","score":0.5740063190460205},{"id":"https://openalex.org/keywords/relevance","display_name":"Relevance (law)","score":0.4356541633605957},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.3921199440956116},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.38538822531700134},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.2180955708026886},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.08584865927696228},{"id":"https://openalex.org/keywords/chemistry","display_name":"Chemistry","score":0.07088503241539001}],"concepts":[{"id":"https://openalex.org/C9893847","wikidata":"https://www.wikidata.org/wiki/Q1425625","display_name":"Reproducibility","level":2,"score":0.8629893064498901},{"id":"https://openalex.org/C2779356329","wikidata":"https://www.wikidata.org/wiki/Q922625","display_name":"Checklist","level":2,"score":0.7388284206390381},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6997463703155518},{"id":"https://openalex.org/C2779010991","wikidata":"https://www.wikidata.org/wiki/Q2720909","display_name":"Artifact (error)","level":2,"score":0.6199007630348206},{"id":"https://openalex.org/C9652623","wikidata":"https://www.wikidata.org/wiki/Q190109","display_name":"Field (mathematics)","level":2,"score":0.5740063190460205},{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.4356541633605957},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.3921199440956116},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.38538822531700134},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2180955708026886},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.08584865927696228},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.07088503241539001},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.0},{"id":"https://openalex.org/C43617362","wikidata":"https://www.wikidata.org/wiki/Q170050","display_name":"Chromatography","level":1,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C202444582","wikidata":"https://www.wikidata.org/wiki/Q837863","display_name":"Pure mathematics","level":1,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3477495.3532686","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3477495.3532686","pdf_url":null,"source":{"id":"https://openalex.org/S4363608773","display_name":"Proceedings of the 45th International ACM SIGIR Conference on Research and Development in Information Retrieval","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 45th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":17,"referenced_works":["https://openalex.org/W1975004953","https://openalex.org/W2468776465","https://openalex.org/W2754517384","https://openalex.org/W2787225861","https://openalex.org/W2891211322","https://openalex.org/W2891288076","https://openalex.org/W2914639375","https://openalex.org/W2957191877","https://openalex.org/W2964020128","https://openalex.org/W2970043916","https://openalex.org/W3034471540","https://openalex.org/W3105114834","https://openalex.org/W3108226083","https://openalex.org/W3153873661","https://openalex.org/W3162597835","https://openalex.org/W4235887602","https://openalex.org/W4297822669"],"related_works":["https://openalex.org/W973641971","https://openalex.org/W2384888906","https://openalex.org/W2003734918","https://openalex.org/W1584628001","https://openalex.org/W2036641180","https://openalex.org/W2106813246","https://openalex.org/W4210915781","https://openalex.org/W2087637638","https://openalex.org/W2080228689","https://openalex.org/W2003667875"],"abstract_inverted_index":{"While":[0],"recent":[1],"progress":[2],"in":[3,35,87,105],"the":[4,17,32,42,51,79,88],"field":[5],"of":[6,19,44,114],"machine":[7],"learning":[8],"(ML)":[9],"and":[10,66,69,73],"information":[11,34,90],"retrieval":[12],"(IR)":[13],"has":[14],"been":[15],"significant,":[16],"reproducibility":[18,57,71],"these":[20],"cutting-edge":[21],"results":[22],"is":[23,85],"often":[24],"lacking,":[25],"with":[26,107],"many":[27],"submissions":[28],"failing":[29],"to":[30,37,100],"provide":[31],"necessary":[33],"order":[36],"ensure":[38,102],"subsequent":[39],"reproducibility.":[40],"Despite":[41],"introduction":[43,99],"self-check":[45],"mechanisms":[46],"before":[47],"submission":[48],"(such":[49],"as":[50,96],"Reproducibility":[52],"Checklist,":[53],"criteria":[54],"for":[55,81],"evaluating":[56],"during":[58],"reviewing":[59],"at":[60,75],"several":[61],"major":[62,76],"conferences,":[63,78],"artifact":[64],"review":[65],"badging":[67],"framework,":[68],"dedicated":[70],"tracks":[72],"challenges":[74],"IR":[77,115],"motivation":[80],"executing":[82],"reproducible":[83,103],"research":[84,104],"lacking":[86],"broader":[89],"community.":[91],"We":[92],"propose":[93],"this":[94],"tutorial":[95],"a":[97,108],"gentle":[98],"help":[101],"IR,":[106],"specific":[109],"emphasis":[110],"on":[111],"ML":[112],"aspects":[113],"research.":[116]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":1}],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-10-10T00:00:00"}