{"id":"https://openalex.org/W3002955717","doi":"https://doi.org/10.1145/3343413.3378004","title":"Estimating Error and Bias in Offline Evaluation Results","display_name":"Estimating Error and Bias in Offline Evaluation Results","publication_year":2020,"publication_date":"2020-03-12","ids":{"openalex":"https://openalex.org/W3002955717","doi":"https://doi.org/10.1145/3343413.3378004","mag":"3002955717"},"language":"en","primary_location":{"id":"doi:10.1145/3343413.3378004","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3343413.3378004","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3343413.3378004","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2020 Conference on Human Information Interaction and Retrieval","raw_type":"proceedings-article"},"type":"article","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3343413.3378004","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Mucun Tian","orcid":null},"institutions":[{"id":"https://openalex.org/I120156002","display_name":"Boise State University","ror":"https://ror.org/02e3zdp86","country_code":"US","type":"education","lineage":["https://openalex.org/I120156002"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Mucun Tian","raw_affiliation_strings":["Boise State University, Boise, ID, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Boise State University, Boise, ID, USA","institution_ids":["https://openalex.org/I120156002"]}]},{"author_position":"last","author":{"id":null,"display_name":"Michael D. Ekstrand","orcid":null},"institutions":[{"id":"https://openalex.org/I120156002","display_name":"Boise State University","ror":"https://ror.org/02e3zdp86","country_code":"US","type":"education","lineage":["https://openalex.org/I120156002"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Michael D. Ekstrand","raw_affiliation_strings":["Boise State University, Boise, ID, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Boise State University, Boise, ID, USA","institution_ids":["https://openalex.org/I120156002"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I120156002"],"apc_list":null,"apc_paid":null,"fwci":1.6327,"has_fulltext":true,"cited_by_count":9,"citation_normalized_percentile":{"value":0.87557199,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":97},"biblio":{"volume":null,"issue":null,"first_page":"392","last_page":"396"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10203","display_name":"Recommender Systems and Techniques","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10203","display_name":"Recommender Systems and Techniques","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12101","display_name":"Advanced Bandit Algorithms Research","score":0.9969000220298767,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12761","display_name":"Data Stream Mining Techniques","score":0.9876999855041504,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/metric","display_name":"Metric (unit)","score":0.6873999834060669},{"id":"https://openalex.org/keywords/recommender-system","display_name":"Recommender system","score":0.6274999976158142},{"id":"https://openalex.org/keywords/missing-data","display_name":"Missing data","score":0.6252999901771545},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.5198000073432922},{"id":"https://openalex.org/keywords/protocol","display_name":"Protocol (science)","score":0.5098999738693237},{"id":"https://openalex.org/keywords/online-and-offline","display_name":"Online and offline","score":0.38510000705718994},{"id":"https://openalex.org/keywords/order","display_name":"Order (exchange)","score":0.3395000100135803}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7423999905586243},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.6873999834060669},{"id":"https://openalex.org/C557471498","wikidata":"https://www.wikidata.org/wiki/Q554950","display_name":"Recommender system","level":2,"score":0.6274999976158142},{"id":"https://openalex.org/C9357733","wikidata":"https://www.wikidata.org/wiki/Q6878417","display_name":"Missing data","level":2,"score":0.6252999901771545},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.5415999889373779},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5338000059127808},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.5198000073432922},{"id":"https://openalex.org/C2780385302","wikidata":"https://www.wikidata.org/wiki/Q367158","display_name":"Protocol (science)","level":3,"score":0.5098999738693237},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4293000102043152},{"id":"https://openalex.org/C2780102126","wikidata":"https://www.wikidata.org/wiki/Q10928179","display_name":"Online and offline","level":2,"score":0.38510000705718994},{"id":"https://openalex.org/C182306322","wikidata":"https://www.wikidata.org/wiki/Q1779371","display_name":"Order (exchange)","level":2,"score":0.3395000100135803},{"id":"https://openalex.org/C133462117","wikidata":"https://www.wikidata.org/wiki/Q4929239","display_name":"Data collection","level":2,"score":0.31310001015663147},{"id":"https://openalex.org/C77618280","wikidata":"https://www.wikidata.org/wiki/Q1155772","display_name":"Scheme (mathematics)","level":2,"score":0.3109000027179718},{"id":"https://openalex.org/C2780898871","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Performance metric","level":2,"score":0.30630001425743103},{"id":"https://openalex.org/C21569690","wikidata":"https://www.wikidata.org/wiki/Q94702","display_name":"Collaborative filtering","level":3,"score":0.3019999861717224},{"id":"https://openalex.org/C67186912","wikidata":"https://www.wikidata.org/wiki/Q367664","display_name":"Data modeling","level":2,"score":0.2711000144481659},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.26809999346733093},{"id":"https://openalex.org/C96250715","wikidata":"https://www.wikidata.org/wiki/Q965330","display_name":"Estimation","level":2,"score":0.26440000534057617},{"id":"https://openalex.org/C167085575","wikidata":"https://www.wikidata.org/wiki/Q6803654","display_name":"Mean squared prediction error","level":2,"score":0.2612999975681305}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3343413.3378004","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3343413.3378004","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3343413.3378004","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2020 Conference on Human Information Interaction and Retrieval","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2001.09455","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2001.09455","pdf_url":"https://arxiv.org/pdf/2001.09455","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"}],"best_oa_location":{"id":"doi:10.1145/3343413.3378004","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3343413.3378004","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3343413.3378004","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2020 Conference on Human Information Interaction and Retrieval","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G5430501557","display_name":null,"funder_award_id":"17-51278","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3002955717.pdf","grobid_xml":"https://content.openalex.org/works/W3002955717.grobid-xml"},"referenced_works_count":12,"referenced_works":["https://openalex.org/W1835900096","https://openalex.org/W1992665562","https://openalex.org/W2026773017","https://openalex.org/W2097638479","https://openalex.org/W2104827998","https://openalex.org/W2150886314","https://openalex.org/W2741544350","https://openalex.org/W2784068709","https://openalex.org/W2888556752","https://openalex.org/W3210800143","https://openalex.org/W4297971002","https://openalex.org/W6678276431"],"related_works":[],"abstract_inverted_index":{"Offline":[0],"evaluations":[1,20],"of":[2,28,32],"recommender":[3,136],"systems":[4],"attempt":[5],"to":[6,45,64,86,101,122,152],"estimate":[7,87],"users'":[8],"satisfaction":[9],"with":[10,25,154],"recommendations":[11],"using":[12],"static":[13],"data":[14,74,93,111],"from":[15,71],"prior":[16],"user":[17],"interactions.":[18],"These":[19],"provide":[21],"researchers":[22],"and":[23,36,75,105,127],"developers":[24],"first":[26],"approximations":[27],"the":[29,57,65,72,88,113,119],"likely":[30],"performance":[31],"a":[33,83,134,139],"new":[34],"system":[35],"help":[37],"weed":[38],"out":[39],"bad":[40],"ideas":[41],"before":[42],"presenting":[43],"them":[44],"users.":[46],"However,":[47],"offline":[48,156],"evaluation":[49,97,120],"cannot":[50,76],"accurately":[51],"assess":[52,102,153],"novel,":[53],"relevant":[54],"recommendations,":[55],"because":[56],"most":[58],"novel":[59],"items":[60],"were":[61],"previously":[62],"unknown":[63],"user,":[66],"so":[67],"they":[68],"are":[69],"missing":[70,92,110],"historical":[73],"be":[77,150],"judged":[78],"as":[79],"relevant.":[80],"We":[81,107],"present":[82],"simulation":[84],"study":[85],"error":[89],"that":[90,109,133],"such":[91],"causes":[94,118],"in":[95,99,112,128,145],"commonly-used":[96],"metrics":[98],"order":[100],"its":[103],"prevalence":[104],"impact.":[106],"find":[108],"rating":[114],"or":[115],"observation":[116],"process":[117],"protocol":[121],"systematically":[123],"mis-estimate":[124],"metric":[125],"values,":[126],"some":[129],"cases":[130],"erroneously":[131],"determine":[132],"popularity-based":[135],"outperforms":[137],"even":[138],"perfect":[140],"personalized":[141],"recommender.":[142],"Substantial":[143],"breakthroughs":[144],"recommendation":[146],"quality,":[147],"therefore,":[148],"will":[149],"difficult":[151],"existing":[155],"techniques.":[157]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":2},{"year":2023,"cited_by_count":4},{"year":2021,"cited_by_count":1},{"year":2020,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2020-01-30T00:00:00"}