{"id":"https://openalex.org/W4366598314","doi":"https://doi.org/10.1145/3591109","title":"A Framework and Toolkit for Testing the Correctness of Recommendation Algorithms","display_name":"A Framework and Toolkit for Testing the Correctness of Recommendation Algorithms","publication_year":2023,"publication_date":"2023-04-20","ids":{"openalex":"https://openalex.org/W4366598314","doi":"https://doi.org/10.1145/3591109"},"language":"en","primary_location":{"id":"doi:10.1145/3591109","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3591109","pdf_url":null,"source":{"id":"https://openalex.org/S4387284325","display_name":"ACM Transactions on Recommender Systems","issn_l":"2770-6699","issn":["2770-6699"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Recommender Systems","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5079385763","display_name":"Lien Michiels","orcid":"https://orcid.org/0000-0003-0152-2460"},"institutions":[{"id":"https://openalex.org/I149213910","display_name":"University of Antwerp","ror":"https://ror.org/008x57b05","country_code":"BE","type":"education","lineage":["https://openalex.org/I149213910"]}],"countries":["BE"],"is_corresponding":true,"raw_author_name":"Lien Michiels","raw_affiliation_strings":["Froomle, Belgium and University of Antwerp, Antwerp, Belgium"],"raw_orcid":"https://orcid.org/0000-0003-0152-2460","affiliations":[{"raw_affiliation_string":"Froomle, Belgium and University of Antwerp, Antwerp, Belgium","institution_ids":["https://openalex.org/I149213910"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5076099352","display_name":"Robin Verachtert","orcid":"https://orcid.org/0000-0003-0345-7770"},"institutions":[{"id":"https://openalex.org/I149213910","display_name":"University of Antwerp","ror":"https://ror.org/008x57b05","country_code":"BE","type":"education","lineage":["https://openalex.org/I149213910"]}],"countries":["BE"],"is_corresponding":false,"raw_author_name":"Robin Verachtert","raw_affiliation_strings":["Froomle, Belgium and University of Antwerp, Antwerp, Belgium"],"raw_orcid":"https://orcid.org/0000-0003-0345-7770","affiliations":[{"raw_affiliation_string":"Froomle, Belgium and University of Antwerp, Antwerp, Belgium","institution_ids":["https://openalex.org/I149213910"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5050623835","display_name":"Andr\u00e9s Ferraro","orcid":"https://orcid.org/0000-0003-1236-2503"},"institutions":[{"id":"https://openalex.org/I5023651","display_name":"McGill University","ror":"https://ror.org/01pxwe438","country_code":"CA","type":"education","lineage":["https://openalex.org/I5023651"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Andres Ferraro","raw_affiliation_strings":["McGill University, Montr\u00e9al, Canada"],"raw_orcid":"https://orcid.org/0000-0003-1236-2503","affiliations":[{"raw_affiliation_string":"McGill University, Montr\u00e9al, Canada","institution_ids":["https://openalex.org/I5023651"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5048786999","display_name":"Kim Kristin Falk","orcid":"https://orcid.org/0000-0002-3573-9257"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kim Falk","raw_affiliation_strings":["Shopify, Canada"],"raw_orcid":"https://orcid.org/0000-0002-3573-9257","affiliations":[{"raw_affiliation_string":"Shopify, Canada","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5020321444","display_name":"Bart Goethals","orcid":"https://orcid.org/0000-0001-9327-9554"},"institutions":[{"id":"https://openalex.org/I149213910","display_name":"University of Antwerp","ror":"https://ror.org/008x57b05","country_code":"BE","type":"education","lineage":["https://openalex.org/I149213910"]},{"id":"https://openalex.org/I56590836","display_name":"Monash University","ror":"https://ror.org/02bfwt286","country_code":"AU","type":"education","lineage":["https://openalex.org/I56590836"]}],"countries":["AU","BE"],"is_corresponding":false,"raw_author_name":"Bart Goethals","raw_affiliation_strings":["University of Antwerp, Antwerp, Belgium and Monash University, Australia and Froomle, Antwerp, Belgium"],"raw_orcid":"https://orcid.org/0000-0001-9327-9554","affiliations":[{"raw_affiliation_string":"University of Antwerp, Antwerp, Belgium and Monash University, Australia and Froomle, Antwerp, Belgium","institution_ids":["https://openalex.org/I149213910","https://openalex.org/I56590836"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5079385763"],"corresponding_institution_ids":["https://openalex.org/I149213910"],"apc_list":null,"apc_paid":null,"fwci":2.6905,"has_fulltext":false,"cited_by_count":6,"citation_normalized_percentile":{"value":0.91334592,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":97},"biblio":{"volume":"2","issue":"1","first_page":"1","last_page":"45"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10203","display_name":"Recommender Systems and Techniques","score":0.9988999962806702,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10203","display_name":"Recommender Systems and Techniques","score":0.9988999962806702,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12101","display_name":"Advanced Bandit Algorithms Research","score":0.9983999729156494,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.9940000176429749,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8615590333938599},{"id":"https://openalex.org/keywords/test-suite","display_name":"Test suite","score":0.8045251369476318},{"id":"https://openalex.org/keywords/correctness","display_name":"Correctness","score":0.7778739929199219},{"id":"https://openalex.org/keywords/unit-testing","display_name":"Unit testing","score":0.7213789224624634},{"id":"https://openalex.org/keywords/python","display_name":"Python (programming language)","score":0.6389675140380859},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.5895509123802185},{"id":"https://openalex.org/keywords/recommender-system","display_name":"Recommender system","score":0.5808782577514648},{"id":"https://openalex.org/keywords/suite","display_name":"Suite","score":0.5128676891326904},{"id":"https://openalex.org/keywords/white-box-testing","display_name":"White-box testing","score":0.4895676076412201},{"id":"https://openalex.org/keywords/implementation","display_name":"Implementation","score":0.4862741231918335},{"id":"https://openalex.org/keywords/surprise","display_name":"Surprise","score":0.42506492137908936},{"id":"https://openalex.org/keywords/regression-testing","display_name":"Regression testing","score":0.422667533159256},{"id":"https://openalex.org/keywords/code-coverage","display_name":"Code coverage","score":0.4209059476852417},{"id":"https://openalex.org/keywords/integration-testing","display_name":"Integration testing","score":0.4112872779369354},{"id":"https://openalex.org/keywords/test-case","display_name":"Test case","score":0.4067450761795044},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.3565753102302551},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.3371468484401703},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.32514670491218567},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.3100299835205078},{"id":"https://openalex.org/keywords/software-system","display_name":"Software system","score":0.2888246774673462},{"id":"https://openalex.org/keywords/software-construction","display_name":"Software construction","score":0.17231759428977966}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8615590333938599},{"id":"https://openalex.org/C151552104","wikidata":"https://www.wikidata.org/wiki/Q7705809","display_name":"Test suite","level":4,"score":0.8045251369476318},{"id":"https://openalex.org/C55439883","wikidata":"https://www.wikidata.org/wiki/Q360812","display_name":"Correctness","level":2,"score":0.7778739929199219},{"id":"https://openalex.org/C148027188","wikidata":"https://www.wikidata.org/wiki/Q907375","display_name":"Unit testing","level":3,"score":0.7213789224624634},{"id":"https://openalex.org/C519991488","wikidata":"https://www.wikidata.org/wiki/Q28865","display_name":"Python (programming language)","level":2,"score":0.6389675140380859},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.5895509123802185},{"id":"https://openalex.org/C557471498","wikidata":"https://www.wikidata.org/wiki/Q554950","display_name":"Recommender system","level":2,"score":0.5808782577514648},{"id":"https://openalex.org/C79581498","wikidata":"https://www.wikidata.org/wiki/Q1367530","display_name":"Suite","level":2,"score":0.5128676891326904},{"id":"https://openalex.org/C162443782","wikidata":"https://www.wikidata.org/wiki/Q1066228","display_name":"White-box testing","level":5,"score":0.4895676076412201},{"id":"https://openalex.org/C26713055","wikidata":"https://www.wikidata.org/wiki/Q245962","display_name":"Implementation","level":2,"score":0.4862741231918335},{"id":"https://openalex.org/C2780343955","wikidata":"https://www.wikidata.org/wiki/Q333173","display_name":"Surprise","level":2,"score":0.42506492137908936},{"id":"https://openalex.org/C161821725","wikidata":"https://www.wikidata.org/wiki/Q917415","display_name":"Regression testing","level":5,"score":0.422667533159256},{"id":"https://openalex.org/C53942775","wikidata":"https://www.wikidata.org/wiki/Q1211721","display_name":"Code coverage","level":3,"score":0.4209059476852417},{"id":"https://openalex.org/C107683887","wikidata":"https://www.wikidata.org/wiki/Q782466","display_name":"Integration testing","level":3,"score":0.4112872779369354},{"id":"https://openalex.org/C128942645","wikidata":"https://www.wikidata.org/wiki/Q1568346","display_name":"Test case","level":3,"score":0.4067450761795044},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.3565753102302551},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.3371468484401703},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.32514670491218567},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3100299835205078},{"id":"https://openalex.org/C149091818","wikidata":"https://www.wikidata.org/wiki/Q2429814","display_name":"Software system","level":3,"score":0.2888246774673462},{"id":"https://openalex.org/C186846655","wikidata":"https://www.wikidata.org/wiki/Q3398377","display_name":"Software construction","level":4,"score":0.17231759428977966},{"id":"https://openalex.org/C166957645","wikidata":"https://www.wikidata.org/wiki/Q23498","display_name":"Archaeology","level":1,"score":0.0},{"id":"https://openalex.org/C95457728","wikidata":"https://www.wikidata.org/wiki/Q309","display_name":"History","level":0,"score":0.0},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.0},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.0},{"id":"https://openalex.org/C152877465","wikidata":"https://www.wikidata.org/wiki/Q208042","display_name":"Regression analysis","level":2,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3591109","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3591109","pdf_url":null,"source":{"id":"https://openalex.org/S4387284325","display_name":"ACM Transactions on Recommender Systems","issn_l":"2770-6699","issn":["2770-6699"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Recommender Systems","raw_type":"journal-article"},{"id":"pmh:c:irua:197115","is_oa":false,"landing_page_url":"https://hdl.handle.net/10067/1971150151162165141","pdf_url":null,"source":{"id":"https://openalex.org/S4306401849","display_name":"Institutional Repository University of Antwerp (University of Antwerp)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I149213910","host_organization_name":"University of Antwerp","host_organization_lineage":["https://openalex.org/I149213910"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":""}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":72,"referenced_works":["https://openalex.org/W1977290284","https://openalex.org/W1985476803","https://openalex.org/W1987431925","https://openalex.org/W1989279326","https://openalex.org/W1992637661","https://openalex.org/W2014714192","https://openalex.org/W2026034143","https://openalex.org/W2041713059","https://openalex.org/W2054141820","https://openalex.org/W2101409192","https://openalex.org/W2117756735","https://openalex.org/W2136944193","https://openalex.org/W2142144955","https://openalex.org/W2153274408","https://openalex.org/W2154772527","https://openalex.org/W2217442125","https://openalex.org/W2219888463","https://openalex.org/W2324595780","https://openalex.org/W2481908490","https://openalex.org/W2496388318","https://openalex.org/W2516364559","https://openalex.org/W2591957553","https://openalex.org/W2605350416","https://openalex.org/W2626454364","https://openalex.org/W2767696711","https://openalex.org/W2782311202","https://openalex.org/W2782864149","https://openalex.org/W2795199972","https://openalex.org/W2886209086","https://openalex.org/W2912745432","https://openalex.org/W2953831886","https://openalex.org/W2957191877","https://openalex.org/W2963085847","https://openalex.org/W2972520532","https://openalex.org/W2980167976","https://openalex.org/W2982138961","https://openalex.org/W2987577970","https://openalex.org/W2990138404","https://openalex.org/W2997591727","https://openalex.org/W2997617192","https://openalex.org/W3007157104","https://openalex.org/W3008797115","https://openalex.org/W3035507081","https://openalex.org/W3040516071","https://openalex.org/W3040911419","https://openalex.org/W3047010778","https://openalex.org/W3086769137","https://openalex.org/W3087720408","https://openalex.org/W3098638686","https://openalex.org/W3101063193","https://openalex.org/W3101708421","https://openalex.org/W3101793783","https://openalex.org/W3102619277","https://openalex.org/W3105114834","https://openalex.org/W3115386848","https://openalex.org/W3122085188","https://openalex.org/W3125645198","https://openalex.org/W3134063549","https://openalex.org/W3154190849","https://openalex.org/W3200660384","https://openalex.org/W3211255867","https://openalex.org/W3216415775","https://openalex.org/W4206112135","https://openalex.org/W4212774754","https://openalex.org/W4226037318","https://openalex.org/W4283018239","https://openalex.org/W4285806565","https://openalex.org/W4292092796","https://openalex.org/W4296604375","https://openalex.org/W4296604448","https://openalex.org/W4301213493","https://openalex.org/W4387618416"],"related_works":["https://openalex.org/W1861739215","https://openalex.org/W4235263786","https://openalex.org/W2188932121","https://openalex.org/W2395905280","https://openalex.org/W2063289013","https://openalex.org/W2003200839","https://openalex.org/W2284280744","https://openalex.org/W2355117330","https://openalex.org/W2153601374","https://openalex.org/W4253994860"],"abstract_inverted_index":{"Evaluating":[0],"recommender":[1,31,151],"systems":[2],"adequately":[3],"and":[4,18,44,65,88,91,99,111,123,159,184],"thoroughly":[5],"is":[6],"an":[7,133],"important":[8],"task.":[9],"Significant":[10],"efforts":[11],"are":[12,176],"dedicated":[13],"to":[14,60,63,80,144,179],"proposing":[15],"metrics,":[16],"methods,":[17],"protocols":[19],"for":[20,150,203],"doing":[21],"so.":[22],"However,":[23],"there":[24],"has":[25],"been":[26],"little":[27],"discussion":[28],"in":[29,68],"the":[30,35,48,82,97,163,194],"systems\u2019":[32],"literature":[33],"on":[34],"topic":[36],"of":[37,84,101,118,167,169,196],"testing.":[38],"In":[39],"this":[40],"work,":[41],"we":[42,128,172,175],"adopt":[43],"adapt":[45],"concepts":[46],"from":[47],"software":[49],"testing":[50,193],"domain,":[51],"e.g.,":[52],"code":[53],"coverage,":[54],"metamorphic":[55],"testing,":[56,59],"or":[57],"property-based":[58],"help":[61],"researchers":[62],"detect":[64],"correct":[66,92],"faults":[67],"recommendation":[69,86,197,205],"algorithms.":[70,103,206],"We":[71,141],"propose":[72],"a":[73,85],"test":[74,105,139,145,165],"suite":[75,106],"that":[76,94,174,192],"can":[77,95,199],"be":[78],"used":[79],"validate":[81],"correctness":[83,195],"algorithm,":[87],"thus":[89],"identify":[90],"issues":[93],"affect":[96],"performance":[98],"behavior":[100],"these":[102,170],"Our":[104],"contains":[107],"both":[108],"black":[109],"box":[110,113],"white":[112],"tests":[114],"at":[115],"every":[116],"level":[117],"abstraction,":[119],"i.e.,":[120],"system,":[121],"integration,":[122],"unit.":[124],"To":[125],"facilitate":[126],"adoption,":[127],"release":[129],"RecPack":[130,153],"Tests":[131],",":[132,154,156,158],"open-source":[134],"Python":[135,148],"package":[136],"containing":[137],"template":[138],"implementations.":[140],"use":[142],"it":[143],"four":[146],"popular":[147],"packages":[149],"systems:":[152],"PyLensKit":[155],"Surprise":[157],"Cornac":[160],".":[161],"Despite":[162],"high":[164],"coverage":[166],"each":[168],"packages,":[171],"find":[173],"still":[177],"able":[178],"uncover":[180],"undocumented":[181],"functional":[182],"requirements":[183],"even":[185],"some":[186],"bugs.":[187],"This":[188],"validates":[189],"our":[190],"thesis":[191],"algorithms":[198],"complement":[200],"traditional":[201],"methods":[202],"evaluating":[204]},"counts_by_year":[{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":3},{"year":2023,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}