{"id":"https://openalex.org/W3034718811","doi":"https://doi.org/10.1145/3397271.3401096","title":"Agreement and Disagreement between True and False-Positive Metrics in Recommender Systems Evaluation","display_name":"Agreement and Disagreement between True and False-Positive Metrics in Recommender Systems Evaluation","publication_year":2020,"publication_date":"2020-07-25","ids":{"openalex":"https://openalex.org/W3034718811","doi":"https://doi.org/10.1145/3397271.3401096","mag":"3034718811"},"language":"en","primary_location":{"id":"doi:10.1145/3397271.3401096","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3397271.3401096","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 43rd International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5036688806","display_name":"Elisa Mena-Maldonado","orcid":"https://orcid.org/0000-0003-1723-1916"},"institutions":[{"id":"https://openalex.org/I82951845","display_name":"RMIT University","ror":"https://ror.org/04ttjf776","country_code":"AU","type":"education","lineage":["https://openalex.org/I82951845"]}],"countries":["AU"],"is_corresponding":true,"raw_author_name":"Elisa Mena-Maldonado","raw_affiliation_strings":["RMIT University, Melbourne, Australia"],"affiliations":[{"raw_affiliation_string":"RMIT University, Melbourne, Australia","institution_ids":["https://openalex.org/I82951845"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5081476292","display_name":"Roc\u00edo Ca\u00f1amares","orcid":"https://orcid.org/0000-0002-2278-0445"},"institutions":[{"id":"https://openalex.org/I63634437","display_name":"Universidad Aut\u00f3noma de Madrid","ror":"https://ror.org/01cby8j38","country_code":"ES","type":"education","lineage":["https://openalex.org/I63634437"]}],"countries":["ES"],"is_corresponding":false,"raw_author_name":"Roc\u00edo Ca\u00f1amares","raw_affiliation_strings":["Universidad Aut\u00f3noma de Madrid, Madrid, Spain"],"affiliations":[{"raw_affiliation_string":"Universidad Aut\u00f3noma de Madrid, Madrid, Spain","institution_ids":["https://openalex.org/I63634437"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5052940596","display_name":"Pablo Castells","orcid":"https://orcid.org/0000-0003-0668-6317"},"institutions":[{"id":"https://openalex.org/I63634437","display_name":"Universidad Aut\u00f3noma de Madrid","ror":"https://ror.org/01cby8j38","country_code":"ES","type":"education","lineage":["https://openalex.org/I63634437"]}],"countries":["ES"],"is_corresponding":false,"raw_author_name":"Pablo Castells","raw_affiliation_strings":["Universidad Aut\u00f3noma de Madrid, Madrid, Spain"],"affiliations":[{"raw_affiliation_string":"Universidad Aut\u00f3noma de Madrid, Madrid, Spain","institution_ids":["https://openalex.org/I63634437"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5039040673","display_name":"Yongli Ren","orcid":"https://orcid.org/0000-0002-3137-9653"},"institutions":[{"id":"https://openalex.org/I82951845","display_name":"RMIT University","ror":"https://ror.org/04ttjf776","country_code":"AU","type":"education","lineage":["https://openalex.org/I82951845"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Yongli Ren","raw_affiliation_strings":["RMIT University, Melbourne, Australia"],"affiliations":[{"raw_affiliation_string":"RMIT University, Melbourne, Australia","institution_ids":["https://openalex.org/I82951845"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5072614956","display_name":"Mark Sanderson","orcid":"https://orcid.org/0000-0003-0487-9609"},"institutions":[{"id":"https://openalex.org/I82951845","display_name":"RMIT University","ror":"https://ror.org/04ttjf776","country_code":"AU","type":"education","lineage":["https://openalex.org/I82951845"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Mark Sanderson","raw_affiliation_strings":["RMIT University, Melbourne, Australia"],"affiliations":[{"raw_affiliation_string":"RMIT University, Melbourne, Australia","institution_ids":["https://openalex.org/I82951845"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5036688806"],"corresponding_institution_ids":["https://openalex.org/I82951845"],"apc_list":null,"apc_paid":null,"fwci":2.9359,"has_fulltext":false,"cited_by_count":17,"citation_normalized_percentile":{"value":0.92614071,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"841","last_page":"850"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10203","display_name":"Recommender Systems and Techniques","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10203","display_name":"Recommender Systems and Techniques","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12101","display_name":"Advanced Bandit Algorithms Research","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11165","display_name":"Image and Video Quality Assessment","score":0.9782000184059143,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/popularity","display_name":"Popularity","score":0.8169839978218079},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7157120704650879},{"id":"https://openalex.org/keywords/complement","display_name":"Complement (music)","score":0.6458664536476135},{"id":"https://openalex.org/keywords/recommender-system","display_name":"Recommender system","score":0.6440860629081726},{"id":"https://openalex.org/keywords/metric","display_name":"Metric (unit)","score":0.5263525247573853},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.4556870460510254},{"id":"https://openalex.org/keywords/false-positive-rate","display_name":"False positive rate","score":0.4217718243598938},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.36118805408477783},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.3420841693878174},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3012590706348419},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.22451290488243103},{"id":"https://openalex.org/keywords/social-psychology","display_name":"Social psychology","score":0.1459784209728241},{"id":"https://openalex.org/keywords/epistemology","display_name":"Epistemology","score":0.07234442234039307}],"concepts":[{"id":"https://openalex.org/C2780586970","wikidata":"https://www.wikidata.org/wiki/Q1357284","display_name":"Popularity","level":2,"score":0.8169839978218079},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7157120704650879},{"id":"https://openalex.org/C112313634","wikidata":"https://www.wikidata.org/wiki/Q7886648","display_name":"Complement (music)","level":5,"score":0.6458664536476135},{"id":"https://openalex.org/C557471498","wikidata":"https://www.wikidata.org/wiki/Q554950","display_name":"Recommender system","level":2,"score":0.6440860629081726},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.5263525247573853},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.4556870460510254},{"id":"https://openalex.org/C95922358","wikidata":"https://www.wikidata.org/wiki/Q5432725","display_name":"False positive rate","level":2,"score":0.4217718243598938},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.36118805408477783},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3420841693878174},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3012590706348419},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.22451290488243103},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.1459784209728241},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.07234442234039307},{"id":"https://openalex.org/C55493867","wikidata":"https://www.wikidata.org/wiki/Q7094","display_name":"Biochemistry","level":1,"score":0.0},{"id":"https://openalex.org/C188082640","wikidata":"https://www.wikidata.org/wiki/Q1780899","display_name":"Complementation","level":4,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C127716648","wikidata":"https://www.wikidata.org/wiki/Q104053","display_name":"Phenotype","level":3,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C104317684","wikidata":"https://www.wikidata.org/wiki/Q7187","display_name":"Gene","level":2,"score":0.0},{"id":"https://openalex.org/C21547014","wikidata":"https://www.wikidata.org/wiki/Q1423657","display_name":"Operations management","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3397271.3401096","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3397271.3401096","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 43rd International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},{"id":"pmh:oai:alma.61RMIT_INST:11273008310001341","is_oa":false,"landing_page_url":"http://purl.org/au-research/grants/arc/DP190101485","pdf_url":null,"source":{"id":"https://openalex.org/S4306402074","display_name":"RMIT Research Repository (RMIT University Library)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I82951845","host_organization_name":"RMIT University","host_organization_lineage":["https://openalex.org/I82951845"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.5199999809265137,"id":"https://metadata.un.org/sdg/16","display_name":"Peace, Justice and strong institutions"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":46,"referenced_works":["https://openalex.org/W115504948","https://openalex.org/W119627244","https://openalex.org/W153313452","https://openalex.org/W1224564842","https://openalex.org/W1967879792","https://openalex.org/W1971040550","https://openalex.org/W1987431925","https://openalex.org/W1992665562","https://openalex.org/W1994389483","https://openalex.org/W2006323481","https://openalex.org/W2020631728","https://openalex.org/W2038531350","https://openalex.org/W2045745608","https://openalex.org/W2049455633","https://openalex.org/W2049670925","https://openalex.org/W2062543652","https://openalex.org/W2071111773","https://openalex.org/W2097638479","https://openalex.org/W2101409192","https://openalex.org/W2109244020","https://openalex.org/W2110672659","https://openalex.org/W2112285779","https://openalex.org/W2123528936","https://openalex.org/W2131381130","https://openalex.org/W2158810025","https://openalex.org/W2171960770","https://openalex.org/W2187424591","https://openalex.org/W2213191543","https://openalex.org/W2340502990","https://openalex.org/W2730215775","https://openalex.org/W2739587313","https://openalex.org/W2740253077","https://openalex.org/W2799048248","https://openalex.org/W2892391771","https://openalex.org/W2892888989","https://openalex.org/W2911829170","https://openalex.org/W2949703874","https://openalex.org/W3105971015","https://openalex.org/W3125944471","https://openalex.org/W4226280022","https://openalex.org/W4300958906","https://openalex.org/W6606837198","https://openalex.org/W6662052390","https://openalex.org/W6667409676","https://openalex.org/W6687422623","https://openalex.org/W6713979495"],"related_works":["https://openalex.org/W2368605798","https://openalex.org/W2518037665","https://openalex.org/W2348524959","https://openalex.org/W2477036161","https://openalex.org/W2368049389","https://openalex.org/W2384861574","https://openalex.org/W2170801710","https://openalex.org/W2952704802","https://openalex.org/W4294565801","https://openalex.org/W2949545456"],"abstract_inverted_index":{"False-positive":[0],"metrics":[1,26,45,51,89,101,118],"can":[2],"capture":[3],"an":[4,82],"important":[5],"side":[6],"of":[7,14,24,32,56,64,93,107,111],"recommendation":[8,110],"quality,":[9],"focusing":[10],"on":[11],"the":[12,30,40,53,75,85,88,91,109,120,129],"impact":[13,97],"suggestions":[15],"that":[16,27,67],"are":[17],"disliked":[18],"by":[19,77],"users,":[20],"as":[21,114],"a":[22,61,133],"complement":[23],"common":[25],"only":[28],"measure":[29],"amount":[31],"successful":[33],"recommendations.":[34],"In":[35],"this":[36],"paper":[37],"we":[38,140],"research":[39],"extent":[41],"to":[42],"which":[43,96,139],"false-positive":[44,117],"agree":[46],"or":[47],"disagree":[48],"with":[49,115,132,147],"true-positive":[50,100],"in":[52,74,90,102,128,145],"offline":[54],"evaluation":[55],"recommender":[57],"systems.":[58],"We":[59,80,122],"discover":[60],"surprising":[62],"degree":[63],"systematic":[65],"disagreement":[66],"was":[68],"occasionally":[69],"noted":[70],"but":[71],"not":[72],"explained":[73],"literature":[76],"previous":[78],"authors.":[79],"find":[81],"explanation":[83,135],"for":[84,136],"discrepancy":[86],"be-tween":[87],"effect":[92],"popularity":[94],"biases,":[95],"false":[98],"and":[99,126,142],"very":[103],"different":[104,148],"ways:":[105],"instead":[106],"rewarding":[108],"popular":[112],"items,":[113],"true-positive,":[116],"penalize":[119],"popular.":[121],"determine":[123],"precise":[124],"conditions":[125],"cases":[127],"general":[130],"trends,":[131],"formal":[134],"our":[137],"findings,":[138],"confirm":[141],"illustrate":[143],"empirically":[144],"experiments":[146],"datasets.":[149]},"counts_by_year":[{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":4},{"year":2023,"cited_by_count":2},{"year":2022,"cited_by_count":6},{"year":2021,"cited_by_count":3}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}