{"id":"https://openalex.org/W3134063549","doi":"https://doi.org/10.1145/3404835.3463245","title":"Elliot: A Comprehensive and Rigorous Framework for Reproducible Recommender Systems Evaluation","display_name":"Elliot: A Comprehensive and Rigorous Framework for Reproducible Recommender Systems Evaluation","publication_year":2021,"publication_date":"2021-07-11","ids":{"openalex":"https://openalex.org/W3134063549","doi":"https://doi.org/10.1145/3404835.3463245","mag":"3134063549"},"language":"en","primary_location":{"id":"doi:10.1145/3404835.3463245","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3404835.3463245","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 44th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},"type":"article","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2103.02590","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Vito Walter Anelli","orcid":null},"institutions":[{"id":"https://openalex.org/I68618741","display_name":"Polytechnic University of Bari","ror":"https://ror.org/03c44v465","country_code":"IT","type":"education","lineage":["https://openalex.org/I68618741"]}],"countries":["IT"],"is_corresponding":true,"raw_author_name":"Vito Walter Anelli","raw_affiliation_strings":["Politecnico di Bari, Bari, Italy"],"affiliations":[{"raw_affiliation_string":"Politecnico di Bari, Bari, Italy","institution_ids":["https://openalex.org/I68618741"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Alejandro Bellogin","orcid":null},"institutions":[{"id":"https://openalex.org/I63634437","display_name":"Universidad Aut\u00f3noma de Madrid","ror":"https://ror.org/01cby8j38","country_code":"ES","type":"education","lineage":["https://openalex.org/I63634437"]}],"countries":["ES"],"is_corresponding":false,"raw_author_name":"Alejandro Bellogin","raw_affiliation_strings":["Aut\u00f3noma Madrid, Madrid, Spain"],"affiliations":[{"raw_affiliation_string":"Aut\u00f3noma Madrid, Madrid, Spain","institution_ids":["https://openalex.org/I63634437"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Antonio Ferrara","orcid":null},"institutions":[{"id":"https://openalex.org/I68618741","display_name":"Polytechnic University of Bari","ror":"https://ror.org/03c44v465","country_code":"IT","type":"education","lineage":["https://openalex.org/I68618741"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Antonio Ferrara","raw_affiliation_strings":["Politecnico di Bari, Bari, Italy"],"affiliations":[{"raw_affiliation_string":"Politecnico di Bari, Bari, Italy","institution_ids":["https://openalex.org/I68618741"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Daniele Malitesta","orcid":null},"institutions":[{"id":"https://openalex.org/I68618741","display_name":"Polytechnic University of Bari","ror":"https://ror.org/03c44v465","country_code":"IT","type":"education","lineage":["https://openalex.org/I68618741"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Daniele Malitesta","raw_affiliation_strings":["Politecnico di Bari, Bari, Italy"],"affiliations":[{"raw_affiliation_string":"Politecnico di Bari, Bari, Italy","institution_ids":["https://openalex.org/I68618741"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Felice Antonio Merra","orcid":null},"institutions":[{"id":"https://openalex.org/I68618741","display_name":"Polytechnic University of Bari","ror":"https://ror.org/03c44v465","country_code":"IT","type":"education","lineage":["https://openalex.org/I68618741"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Felice Antonio Merra","raw_affiliation_strings":["Politecnico di Bari, Bari, Italy"],"affiliations":[{"raw_affiliation_string":"Politecnico di Bari, Bari, Italy","institution_ids":["https://openalex.org/I68618741"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Claudio Pomo","orcid":null},"institutions":[{"id":"https://openalex.org/I68618741","display_name":"Polytechnic University of Bari","ror":"https://ror.org/03c44v465","country_code":"IT","type":"education","lineage":["https://openalex.org/I68618741"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Claudio Pomo","raw_affiliation_strings":["Politecnico di Bari, Bari, Italy"],"affiliations":[{"raw_affiliation_string":"Politecnico di Bari, Bari, Italy","institution_ids":["https://openalex.org/I68618741"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Francesco Maria Donini","orcid":null},"institutions":[{"id":"https://openalex.org/I138938424","display_name":"Universit\u00e0 degli Studi della Tuscia","ror":"https://ror.org/03svwq685","country_code":"IT","type":"education","lineage":["https://openalex.org/I138938424"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Francesco Maria Donini","raw_affiliation_strings":["Universit\u00e0 della Tuscia, Viterbo, Italy"],"affiliations":[{"raw_affiliation_string":"Universit\u00e0 della Tuscia, Viterbo, Italy","institution_ids":["https://openalex.org/I138938424"]}]},{"author_position":"last","author":{"id":null,"display_name":"Tommaso Di Noia","orcid":null},"institutions":[{"id":"https://openalex.org/I68618741","display_name":"Polytechnic University of Bari","ror":"https://ror.org/03c44v465","country_code":"IT","type":"education","lineage":["https://openalex.org/I68618741"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Tommaso Di Noia","raw_affiliation_strings":["Politecnico di Bari, Bari, Italy"],"affiliations":[{"raw_affiliation_string":"Politecnico di Bari, Bari, Italy","institution_ids":["https://openalex.org/I68618741"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":8,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I68618741"],"apc_list":null,"apc_paid":null,"fwci":15.8964,"has_fulltext":false,"cited_by_count":88,"citation_normalized_percentile":{"value":0.99068768,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":97,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"2405","last_page":"2414"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10203","display_name":"Recommender Systems and Techniques","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10203","display_name":"Recommender Systems and Techniques","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12101","display_name":"Advanced Bandit Algorithms Research","score":0.9976000189781189,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11165","display_name":"Image and Video Quality Assessment","score":0.9914000034332275,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/recommender-system","display_name":"Recommender system","score":0.7181000113487244},{"id":"https://openalex.org/keywords/hyperparameter","display_name":"Hyperparameter","score":0.7039999961853027},{"id":"https://openalex.org/keywords/pipeline","display_name":"Pipeline (software)","score":0.6195999979972839},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.5843999981880188},{"id":"https://openalex.org/keywords/data-set","display_name":"Data set","score":0.3887999951839447},{"id":"https://openalex.org/keywords/simple","display_name":"Simple (philosophy)","score":0.382999986410141}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8051999807357788},{"id":"https://openalex.org/C557471498","wikidata":"https://www.wikidata.org/wiki/Q554950","display_name":"Recommender system","level":2,"score":0.7181000113487244},{"id":"https://openalex.org/C8642999","wikidata":"https://www.wikidata.org/wiki/Q4171168","display_name":"Hyperparameter","level":2,"score":0.7039999961853027},{"id":"https://openalex.org/C43521106","wikidata":"https://www.wikidata.org/wiki/Q2165493","display_name":"Pipeline (software)","level":2,"score":0.6195999979972839},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.5843999981880188},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.5467000007629395},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.516700029373169},{"id":"https://openalex.org/C58489278","wikidata":"https://www.wikidata.org/wiki/Q1172284","display_name":"Data set","level":2,"score":0.3887999951839447},{"id":"https://openalex.org/C2780586882","wikidata":"https://www.wikidata.org/wiki/Q7520643","display_name":"Simple (philosophy)","level":2,"score":0.382999986410141},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3714999854564667},{"id":"https://openalex.org/C21569690","wikidata":"https://www.wikidata.org/wiki/Q94702","display_name":"Collaborative filtering","level":3,"score":0.3578000068664551},{"id":"https://openalex.org/C115051666","wikidata":"https://www.wikidata.org/wiki/Q6522493","display_name":"Ranging","level":2,"score":0.32749998569488525},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.30219998955726624},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.28700000047683716},{"id":"https://openalex.org/C114289077","wikidata":"https://www.wikidata.org/wiki/Q3284399","display_name":"Statistical model","level":2,"score":0.27079999446868896},{"id":"https://openalex.org/C2986587452","wikidata":"https://www.wikidata.org/wiki/Q938438","display_name":"Statistical analysis","level":2,"score":0.2702000141143799},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.26589998602867126},{"id":"https://openalex.org/C132964779","wikidata":"https://www.wikidata.org/wiki/Q2110223","display_name":"Raw data","level":2,"score":0.25459998846054077}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3404835.3463245","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3404835.3463245","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 44th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2103.02590","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2103.02590","pdf_url":"https://arxiv.org/pdf/2103.02590","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2103.02590","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2103.02590","pdf_url":"https://arxiv.org/pdf/2103.02590","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":63,"referenced_works":["https://openalex.org/W1479822238","https://openalex.org/W1720514416","https://openalex.org/W1965355809","https://openalex.org/W1967787801","https://openalex.org/W1994389483","https://openalex.org/W2031237011","https://openalex.org/W2042281163","https://openalex.org/W2049750575","https://openalex.org/W2054141820","https://openalex.org/W2062959588","https://openalex.org/W2072992969","https://openalex.org/W2078488685","https://openalex.org/W2108920354","https://openalex.org/W2119523409","https://openalex.org/W2119825970","https://openalex.org/W2135598826","https://openalex.org/W2140942692","https://openalex.org/W2144487656","https://openalex.org/W2144807535","https://openalex.org/W2150886314","https://openalex.org/W2152208379","https://openalex.org/W2155106456","https://openalex.org/W2161351857","https://openalex.org/W2253995343","https://openalex.org/W2475334473","https://openalex.org/W2509235963","https://openalex.org/W2515144511","https://openalex.org/W2604433096","https://openalex.org/W2604662567","https://openalex.org/W2605350416","https://openalex.org/W2619206542","https://openalex.org/W2723293840","https://openalex.org/W2740920897","https://openalex.org/W2741249238","https://openalex.org/W2742143754","https://openalex.org/W2748058847","https://openalex.org/W2783272285","https://openalex.org/W2783279085","https://openalex.org/W2783603395","https://openalex.org/W2802187397","https://openalex.org/W2892923641","https://openalex.org/W2897167574","https://openalex.org/W2897363137","https://openalex.org/W2945827670","https://openalex.org/W2957191877","https://openalex.org/W2958300421","https://openalex.org/W2962712142","https://openalex.org/W2962907114","https://openalex.org/W2963085847","https://openalex.org/W2963323306","https://openalex.org/W2971522550","https://openalex.org/W2971600245","https://openalex.org/W2972941122","https://openalex.org/W2981109095","https://openalex.org/W2982267219","https://openalex.org/W3035446616","https://openalex.org/W3040516071","https://openalex.org/W3045200674","https://openalex.org/W3047010778","https://openalex.org/W3088444111","https://openalex.org/W3088694469","https://openalex.org/W3088924330","https://openalex.org/W3115418111"],"related_works":[],"abstract_inverted_index":{"Recommender":[0],"Systems":[1],"have":[2,61],"shown":[3],"to":[4,9,66,78,117,149],"be":[5],"an":[6,63,82],"effective":[7],"way":[8],"alleviate":[10],"the":[11,21,46,98,132,138],"over-choice":[12],"problem":[13],"and":[14,17,33,43,57,80,96,109,152,154,159],"provide":[15],"accurate":[16],"tailored":[18],"recommendations.":[19],"However,":[20],"impressive":[22],"number":[23],"of":[24,49,104],"proposed":[25],"recommendation":[26,74,128],"algorithms,":[27],"splitting":[28,107,116],"strategies,":[29],"evaluation":[30,39,51,58],"protocols,":[31],"metrics,":[32],"tasks,":[34],"has":[35],"made":[36],"rigorous":[37],"experimental":[38,53,84],"particularly":[40],"challenging.":[41],"Puzzled":[42],"frustrated":[44],"by":[45,86],"continuous":[47],"recreation":[48],"appropriate":[50],"benchmarks,":[52],"pipelines,":[54],"hyperparameter":[55],"optimization,":[56],"procedures,":[59],"we":[60],"developed":[62],"exhaustive":[64],"framework":[65,75,93],"address":[67],"such":[68],"needs.":[69],"Elliot":[70],"is":[71],"a":[72,88,101],"comprehensive":[73],"that":[76],"aims":[77],"run":[79],"reproduce":[81],"entire":[83],"pipeline":[85],"processing":[87],"simple":[89],"configuration":[90],"file.":[91],"The":[92],"loads,":[94],"filters,":[95],"splits":[97],"data":[99],"considering":[100],"vast":[102],"set":[103],"strategies":[105],"(13":[106],"methods":[108],"8":[110],"filtering":[111],"approaches,":[112],"from":[113,147],"temporal":[114],"training-test":[115],"nested":[118],"K-folds":[119],"Cross-Validation).":[120],"Elliot(https://github.com/sisinflab/elliot)":[121],"optimizes":[122],"hyperparameters":[123],"(51":[124],"strategies)":[125],"for":[126],"several":[127],"algorithms":[129],"(50),":[130],"selects":[131],"best":[133],"models,":[134],"compares":[135],"them":[136],"with":[137],"baselines":[139],"providing":[140],"intra-model":[141],"statistics,":[142],"computes":[143],"metrics":[144],"(36)":[145],"spanning":[146],"accuracy":[148],"beyond-accuracy,":[150],"bias,":[151],"fairness,":[153],"conducts":[155],"statistical":[156],"analysis":[157],"(Wilcoxon":[158],"Paired":[160],"t-test).":[161]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":30},{"year":2024,"cited_by_count":18},{"year":2023,"cited_by_count":16},{"year":2022,"cited_by_count":17},{"year":2021,"cited_by_count":5}],"updated_date":"2026-03-24T08:02:53.985720","created_date":"2021-03-15T00:00:00"}