{"id":"https://openalex.org/W2892888989","doi":"https://doi.org/10.1145/3240323.3240355","title":"Unbiased offline recommender evaluation for missing-not-at-random implicit feedback","display_name":"Unbiased offline recommender evaluation for missing-not-at-random implicit feedback","publication_year":2018,"publication_date":"2018-09-27","ids":{"openalex":"https://openalex.org/W2892888989","doi":"https://doi.org/10.1145/3240323.3240355","mag":"2892888989"},"language":"en","primary_location":{"id":"doi:10.1145/3240323.3240355","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3240323.3240355","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3240323.3240355","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 12th ACM Conference on Recommender Systems","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3240323.3240355","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5057330200","display_name":"Longqi Yang","orcid":"https://orcid.org/0000-0002-6615-8615"},"institutions":[{"id":"https://openalex.org/I205783295","display_name":"Cornell University","ror":"https://ror.org/05bnh6r87","country_code":"US","type":"education","lineage":["https://openalex.org/I205783295"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Longqi Yang","raw_affiliation_strings":["Cornell University"],"affiliations":[{"raw_affiliation_string":"Cornell University","institution_ids":["https://openalex.org/I205783295"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5041864624","display_name":"Yin Cui","orcid":"https://orcid.org/0000-0003-0070-5118"},"institutions":[{"id":"https://openalex.org/I205783295","display_name":"Cornell University","ror":"https://ror.org/05bnh6r87","country_code":"US","type":"education","lineage":["https://openalex.org/I205783295"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yin Cui","raw_affiliation_strings":["Cornell University"],"affiliations":[{"raw_affiliation_string":"Cornell University","institution_ids":["https://openalex.org/I205783295"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100838958","display_name":"Xuan Yuan","orcid":null},"institutions":[{"id":"https://openalex.org/I205783295","display_name":"Cornell University","ror":"https://ror.org/05bnh6r87","country_code":"US","type":"education","lineage":["https://openalex.org/I205783295"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yuan Xuan","raw_affiliation_strings":["Cornell University"],"affiliations":[{"raw_affiliation_string":"Cornell University","institution_ids":["https://openalex.org/I205783295"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100424705","display_name":"Chenyang Wang","orcid":"https://orcid.org/0000-0002-8706-5354"},"institutions":[{"id":"https://openalex.org/I205783295","display_name":"Cornell University","ror":"https://ror.org/05bnh6r87","country_code":"US","type":"education","lineage":["https://openalex.org/I205783295"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Chenyang Wang","raw_affiliation_strings":["Cornell University"],"affiliations":[{"raw_affiliation_string":"Cornell University","institution_ids":["https://openalex.org/I205783295"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5018609918","display_name":"Serge Belongie","orcid":"https://orcid.org/0000-0002-0388-5217"},"institutions":[{"id":"https://openalex.org/I205783295","display_name":"Cornell University","ror":"https://ror.org/05bnh6r87","country_code":"US","type":"education","lineage":["https://openalex.org/I205783295"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Serge Belongie","raw_affiliation_strings":["Cornell University"],"affiliations":[{"raw_affiliation_string":"Cornell University","institution_ids":["https://openalex.org/I205783295"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5051587086","display_name":"Deborah Estrin","orcid":"https://orcid.org/0000-0001-6477-0096"},"institutions":[{"id":"https://openalex.org/I205783295","display_name":"Cornell University","ror":"https://ror.org/05bnh6r87","country_code":"US","type":"education","lineage":["https://openalex.org/I205783295"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Deborah Estrin","raw_affiliation_strings":["Cornell University"],"affiliations":[{"raw_affiliation_string":"Cornell University","institution_ids":["https://openalex.org/I205783295"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5057330200"],"corresponding_institution_ids":["https://openalex.org/I205783295"],"apc_list":null,"apc_paid":null,"fwci":29.6045,"has_fulltext":true,"cited_by_count":208,"citation_normalized_percentile":{"value":0.99570452,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"279","last_page":"287"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10203","display_name":"Recommender Systems and Techniques","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10203","display_name":"Recommender Systems and Techniques","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12101","display_name":"Advanced Bandit Algorithms Research","score":0.996399998664856,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.96670001745224,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/popularity","display_name":"Popularity","score":0.7647321224212646},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7291436195373535},{"id":"https://openalex.org/keywords/leverage","display_name":"Leverage (statistics)","score":0.6402693390846252},{"id":"https://openalex.org/keywords/estimator","display_name":"Estimator","score":0.5819215178489685},{"id":"https://openalex.org/keywords/recommender-system","display_name":"Recommender system","score":0.5298580527305603},{"id":"https://openalex.org/keywords/missing-data","display_name":"Missing data","score":0.48578953742980957},{"id":"https://openalex.org/keywords/ground-truth","display_name":"Ground truth","score":0.43198198080062866},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.41294994950294495},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.40573498606681824},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.36821112036705017},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.32437416911125183},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.2186659574508667},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.1351768970489502},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.10249444842338562}],"concepts":[{"id":"https://openalex.org/C2780586970","wikidata":"https://www.wikidata.org/wiki/Q1357284","display_name":"Popularity","level":2,"score":0.7647321224212646},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7291436195373535},{"id":"https://openalex.org/C153083717","wikidata":"https://www.wikidata.org/wiki/Q6535263","display_name":"Leverage (statistics)","level":2,"score":0.6402693390846252},{"id":"https://openalex.org/C185429906","wikidata":"https://www.wikidata.org/wiki/Q1130160","display_name":"Estimator","level":2,"score":0.5819215178489685},{"id":"https://openalex.org/C557471498","wikidata":"https://www.wikidata.org/wiki/Q554950","display_name":"Recommender system","level":2,"score":0.5298580527305603},{"id":"https://openalex.org/C9357733","wikidata":"https://www.wikidata.org/wiki/Q6878417","display_name":"Missing data","level":2,"score":0.48578953742980957},{"id":"https://openalex.org/C146849305","wikidata":"https://www.wikidata.org/wiki/Q370766","display_name":"Ground truth","level":2,"score":0.43198198080062866},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.41294994950294495},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.40573498606681824},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.36821112036705017},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.32437416911125183},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.2186659574508667},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.1351768970489502},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.10249444842338562},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3240323.3240355","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3240323.3240355","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3240323.3240355","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 12th ACM Conference on Recommender Systems","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3240323.3240355","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3240323.3240355","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3240323.3240355","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 12th ACM Conference on Recommender Systems","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G6683337051","display_name":null,"funder_award_id":"1700832","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G848032724","display_name":null,"funder_award_id":"Science","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320306139","display_name":"Robert Wood Johnson Foundation","ror":"https://ror.org/02ymmdj85"},{"id":"https://openalex.org/F4320318127","display_name":"UnitedHealth Group","ror":"https://ror.org/04a8rt780"},{"id":"https://openalex.org/F4320332161","display_name":"National Institutes of Health","ror":"https://ror.org/01cwqze88"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2892888989.pdf","grobid_xml":"https://content.openalex.org/works/W2892888989.grobid-xml"},"referenced_works_count":28,"referenced_works":["https://openalex.org/W21006490","https://openalex.org/W1991418309","https://openalex.org/W1992665562","https://openalex.org/W2020631728","https://openalex.org/W2026773017","https://openalex.org/W2040367556","https://openalex.org/W2045745608","https://openalex.org/W2046974451","https://openalex.org/W2101409192","https://openalex.org/W2137028279","https://openalex.org/W2137245235","https://openalex.org/W2138909795","https://openalex.org/W2140310134","https://openalex.org/W2157881433","https://openalex.org/W2188353343","https://openalex.org/W2463677609","https://openalex.org/W2507134384","https://openalex.org/W2514896200","https://openalex.org/W2604325789","https://openalex.org/W2604433096","https://openalex.org/W2612658537","https://openalex.org/W2746675896","https://openalex.org/W2748058847","https://openalex.org/W2783603395","https://openalex.org/W2950382198","https://openalex.org/W2952613481","https://openalex.org/W2962694783","https://openalex.org/W2963655167"],"related_works":["https://openalex.org/W2368605798","https://openalex.org/W2518037665","https://openalex.org/W2348524959","https://openalex.org/W2477036161","https://openalex.org/W2368049389","https://openalex.org/W2384861574","https://openalex.org/W2170801710","https://openalex.org/W2952704802","https://openalex.org/W4294565801","https://openalex.org/W4390723067"],"abstract_inverted_index":{"Implicit-feedback":[0],"Recommenders":[1,16],"(ImplicitRec)":[2],"leverage":[3],"positive":[4],"only":[5],"user-item":[6],"interactions,":[7],"such":[8],"as":[9],"clicks,":[10],"to":[11,33,42,131,143],"learn":[12],"personalized":[13],"user":[14],"preferences.":[15],"are":[17,31,39,54],"often":[18],"evaluated":[19],"and":[20,45,48,83,88,108,125,146],"compared":[21],"offline":[22,90],"using":[23,96,104],"datasets":[24,95,107],"collected":[25],"from":[26],"online":[27],"platforms.":[28],"These":[29],"platforms":[30],"subject":[32],"popularity":[34,117],"bias":[35,80,118,129,156],"(i.e.,":[36],"popular":[37],"items":[38],"more":[40,158],"likely":[41],"be":[43],"presented":[44],"interacted":[46],"with),":[47],"therefore":[49],"logged":[50],"ground":[51],"truth":[52],"data":[53,133],"Missing-Not-At-Random":[55],"(MNAR).":[56],"As":[57],"a":[58],"result,":[59],"the":[60,97,148,153,162,169],"widely":[61,110,120],"used":[62,111,142],"Average-Over-All":[63],"(AOA)":[64],"evaluator":[65,91],"is":[66,119,141],"biased":[67],"toward":[68],"accurately":[69],"recommending":[70],"trendy":[71],"items.":[72],"In":[73],"this":[74],"paper,":[75],"we":[76,113],"(a)":[77,116],"investigate":[78],"evaluation":[79,128,155],"of":[81,168],"AOA":[82,140,154],"(b)":[84,127],"develop":[85],"an":[86],"unbiased":[87,149],"practical":[89],"for":[92],"implicit":[93],"MNAR":[94,132],"Inverse-Propensity-Scoring":[98],"(IPS)":[99],"technique.":[100],"Through":[101],"extensive":[102],"experiments":[103],"four":[105,109],"real-world":[106],"algorithms,":[112],"show":[114],"that":[115],"manifested":[121],"in":[122,136,161,166],"item":[123],"presentation":[124],"interaction;":[126],"due":[130],"pervasively":[134],"exists":[135],"most":[137],"cases":[138],"where":[139],"evaluate":[144],"ImplicitRec;":[145],"(c)":[147],"estimator":[150],"significantly":[151],"reduces":[152],"by":[157],"than":[159],"30%":[160],"Yahoo!":[163],"music":[164],"dataset":[165],"terms":[167],"Mean":[170],"Absolute":[171],"Error":[172],"(MAE).":[173]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":20},{"year":2024,"cited_by_count":41},{"year":2023,"cited_by_count":38},{"year":2022,"cited_by_count":35},{"year":2021,"cited_by_count":30},{"year":2020,"cited_by_count":25},{"year":2019,"cited_by_count":17},{"year":2012,"cited_by_count":1}],"updated_date":"2026-04-10T15:06:20.359241","created_date":"2025-10-10T00:00:00"}