{"id":"https://openalex.org/W4386730264","doi":"https://doi.org/10.1145/3604915.3608809","title":"What We Evaluate When We Evaluate Recommender Systems: Understanding Recommender Systems\u2019 Performance using Item Response Theory","display_name":"What We Evaluate When We Evaluate Recommender Systems: Understanding Recommender Systems\u2019 Performance using Item Response Theory","publication_year":2023,"publication_date":"2023-09-14","ids":{"openalex":"https://openalex.org/W4386730264","doi":"https://doi.org/10.1145/3604915.3608809"},"language":"en","primary_location":{"id":"doi:10.1145/3604915.3608809","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3604915.3608809","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3604915.3608809","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 17th ACM Conference on Recommender Systems","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3604915.3608809","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100739482","display_name":"Yang Liu","orcid":"https://orcid.org/0000-0002-1454-9585"},"institutions":[{"id":"https://openalex.org/I133731052","display_name":"University of Helsinki","ror":"https://ror.org/040af2s02","country_code":"FI","type":"education","lineage":["https://openalex.org/I133731052"]}],"countries":["FI"],"is_corresponding":true,"raw_author_name":"Yang Liu","raw_affiliation_strings":["Department of Computer Science, University of Helsinki, Finland"],"raw_orcid":"https://orcid.org/0000-0002-1454-9585","affiliations":[{"raw_affiliation_string":"Department of Computer Science, University of Helsinki, Finland","institution_ids":["https://openalex.org/I133731052"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5007118986","display_name":"Alan Medlar","orcid":"https://orcid.org/0000-0002-5139-9483"},"institutions":[{"id":"https://openalex.org/I133731052","display_name":"University of Helsinki","ror":"https://ror.org/040af2s02","country_code":"FI","type":"education","lineage":["https://openalex.org/I133731052"]}],"countries":["FI"],"is_corresponding":false,"raw_author_name":"Alan Medlar","raw_affiliation_strings":["Department of Computer Science, University of Helsinki, Finland"],"raw_orcid":"https://orcid.org/0000-0002-5139-9483","affiliations":[{"raw_affiliation_string":"Department of Computer Science, University of Helsinki, Finland","institution_ids":["https://openalex.org/I133731052"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5001270670","display_name":"Dorota G\u0142owacka","orcid":"https://orcid.org/0000-0001-7008-3380"},"institutions":[{"id":"https://openalex.org/I133731052","display_name":"University of Helsinki","ror":"https://ror.org/040af2s02","country_code":"FI","type":"education","lineage":["https://openalex.org/I133731052"]}],"countries":["FI"],"is_corresponding":false,"raw_author_name":"Dorota Glowacka","raw_affiliation_strings":["Department of Computer Science, University of Helsinki, Finland"],"raw_orcid":"https://orcid.org/0000-0001-7008-3380","affiliations":[{"raw_affiliation_string":"Department of Computer Science, University of Helsinki, Finland","institution_ids":["https://openalex.org/I133731052"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5100739482"],"corresponding_institution_ids":["https://openalex.org/I133731052"],"apc_list":null,"apc_paid":null,"fwci":2.6905,"has_fulltext":true,"cited_by_count":6,"citation_normalized_percentile":{"value":0.91852455,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"658","last_page":"670"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10203","display_name":"Recommender Systems and Techniques","score":0.9991999864578247,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10203","display_name":"Recommender Systems and Techniques","score":0.9991999864578247,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12101","display_name":"Advanced Bandit Algorithms Research","score":0.9544000029563904,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11161","display_name":"Consumer Market Behavior and Pricing","score":0.935699999332428,"subfield":{"id":"https://openalex.org/subfields/1406","display_name":"Marketing"},"field":{"id":"https://openalex.org/fields/14","display_name":"Business, Management and Accounting"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/recommender-system","display_name":"Recommender system","score":0.8757519125938416},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8031020164489746},{"id":"https://openalex.org/keywords/popularity","display_name":"Popularity","score":0.7866076231002808},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.7178773880004883},{"id":"https://openalex.org/keywords/latent-variable","display_name":"Latent variable","score":0.6095223426818848},{"id":"https://openalex.org/keywords/item-response-theory","display_name":"Item response theory","score":0.5714683532714844},{"id":"https://openalex.org/keywords/rank","display_name":"Rank (graph theory)","score":0.5320428609848022},{"id":"https://openalex.org/keywords/perspective","display_name":"Perspective (graphical)","score":0.47272437810897827},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.45704418420791626},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.4492616653442383},{"id":"https://openalex.org/keywords/latent-variable-model","display_name":"Latent variable model","score":0.41824692487716675},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.38493698835372925},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.31314852833747864},{"id":"https://openalex.org/keywords/psychometrics","display_name":"Psychometrics","score":0.14907726645469666},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.10346195101737976},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.08165019750595093}],"concepts":[{"id":"https://openalex.org/C557471498","wikidata":"https://www.wikidata.org/wiki/Q554950","display_name":"Recommender system","level":2,"score":0.8757519125938416},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8031020164489746},{"id":"https://openalex.org/C2780586970","wikidata":"https://www.wikidata.org/wiki/Q1357284","display_name":"Popularity","level":2,"score":0.7866076231002808},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.7178773880004883},{"id":"https://openalex.org/C51167844","wikidata":"https://www.wikidata.org/wiki/Q4422623","display_name":"Latent variable","level":2,"score":0.6095223426818848},{"id":"https://openalex.org/C19875794","wikidata":"https://www.wikidata.org/wiki/Q1207340","display_name":"Item response theory","level":3,"score":0.5714683532714844},{"id":"https://openalex.org/C164226766","wikidata":"https://www.wikidata.org/wiki/Q7293202","display_name":"Rank (graph theory)","level":2,"score":0.5320428609848022},{"id":"https://openalex.org/C12713177","wikidata":"https://www.wikidata.org/wiki/Q1900281","display_name":"Perspective (graphical)","level":2,"score":0.47272437810897827},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.45704418420791626},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4492616653442383},{"id":"https://openalex.org/C65965080","wikidata":"https://www.wikidata.org/wiki/Q1806885","display_name":"Latent variable model","level":3,"score":0.41824692487716675},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.38493698835372925},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.31314852833747864},{"id":"https://openalex.org/C171606756","wikidata":"https://www.wikidata.org/wiki/Q506132","display_name":"Psychometrics","level":2,"score":0.14907726645469666},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.10346195101737976},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.08165019750595093},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.0},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.0},{"id":"https://openalex.org/C114614502","wikidata":"https://www.wikidata.org/wiki/Q76592","display_name":"Combinatorics","level":1,"score":0.0},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3604915.3608809","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3604915.3608809","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3604915.3608809","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 17th ACM Conference on Recommender Systems","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3604915.3608809","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3604915.3608809","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3604915.3608809","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 17th ACM Conference on Recommender Systems","raw_type":"proceedings-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/10","score":0.7300000190734863,"display_name":"Reduced inequalities"}],"awards":[],"funders":[{"id":"https://openalex.org/F4320322725","display_name":"China Scholarship Council","ror":"https://ror.org/04atp4p48"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4386730264.pdf","grobid_xml":"https://content.openalex.org/works/W4386730264.grobid-xml"},"referenced_works_count":93,"referenced_works":["https://openalex.org/W135984018","https://openalex.org/W178169250","https://openalex.org/W1516111018","https://openalex.org/W1888005072","https://openalex.org/W1971040550","https://openalex.org/W1987431925","https://openalex.org/W2034825773","https://openalex.org/W2038886783","https://openalex.org/W2049670925","https://openalex.org/W2069870183","https://openalex.org/W2090883204","https://openalex.org/W2108920354","https://openalex.org/W2136189984","https://openalex.org/W2142144955","https://openalex.org/W2150886314","https://openalex.org/W2170516266","https://openalex.org/W2253995343","https://openalex.org/W2295739661","https://openalex.org/W2331841392","https://openalex.org/W2475334473","https://openalex.org/W2509235963","https://openalex.org/W2509893387","https://openalex.org/W2548570154","https://openalex.org/W2562236173","https://openalex.org/W2604662567","https://openalex.org/W2605350416","https://openalex.org/W2740253077","https://openalex.org/W2740920897","https://openalex.org/W2767669124","https://openalex.org/W2774321216","https://openalex.org/W2788490371","https://openalex.org/W2793768763","https://openalex.org/W2801992635","https://openalex.org/W2802187397","https://openalex.org/W2888838693","https://openalex.org/W2891340257","https://openalex.org/W2892923641","https://openalex.org/W2898085636","https://openalex.org/W2911778742","https://openalex.org/W2912351665","https://openalex.org/W2912745432","https://openalex.org/W2912981918","https://openalex.org/W2913560138","https://openalex.org/W2945623882","https://openalex.org/W2945827670","https://openalex.org/W2954808392","https://openalex.org/W2962712142","https://openalex.org/W2963085847","https://openalex.org/W2963677523","https://openalex.org/W2963911286","https://openalex.org/W2964052347","https://openalex.org/W2964182926","https://openalex.org/W2970184386","https://openalex.org/W2987577970","https://openalex.org/W2997617192","https://openalex.org/W2998089494","https://openalex.org/W2999649805","https://openalex.org/W3044311607","https://openalex.org/W3045200674","https://openalex.org/W3081170586","https://openalex.org/W3088444111","https://openalex.org/W3089147479","https://openalex.org/W3093945404","https://openalex.org/W3094605801","https://openalex.org/W3098638686","https://openalex.org/W3098723082","https://openalex.org/W3099386565","https://openalex.org/W3100278010","https://openalex.org/W3100324210","https://openalex.org/W3101704389","https://openalex.org/W3101708421","https://openalex.org/W3104030692","https://openalex.org/W3106439716","https://openalex.org/W3125645198","https://openalex.org/W3129482887","https://openalex.org/W3153325943","https://openalex.org/W3153687269","https://openalex.org/W3155919942","https://openalex.org/W3177468934","https://openalex.org/W3185773347","https://openalex.org/W3201149665","https://openalex.org/W3208227120","https://openalex.org/W3209428314","https://openalex.org/W3214743865","https://openalex.org/W4205595445","https://openalex.org/W4220909642","https://openalex.org/W4226037318","https://openalex.org/W4237543597","https://openalex.org/W4283587264","https://openalex.org/W4292414119","https://openalex.org/W4297971002","https://openalex.org/W4327909856","https://openalex.org/W4386729510"],"related_works":["https://openalex.org/W2461917396","https://openalex.org/W2037497866","https://openalex.org/W4243467573","https://openalex.org/W1502435251","https://openalex.org/W62001224","https://openalex.org/W3032390039","https://openalex.org/W1584341211","https://openalex.org/W3122667150","https://openalex.org/W4393387622","https://openalex.org/W4318617505"],"abstract_inverted_index":{"Current":[0],"practices":[1],"in":[2,79,135,169],"offline":[3,88],"evaluation":[4,170],"use":[5,67,91],"rank-based":[6,129],"metrics":[7,130],"to":[8,82,93],"measure":[9],"the":[10,26,29,37,60,96,104,117,143,147,165],"quality":[11],"of":[12,28,39,74,87,99,106,167],"top-n":[13,61,144],"recommendation":[14,101,180],"lists.":[15],"This":[16],"approach":[17],"has":[18],"practical":[19],"benefits":[20],"as":[21],"it":[22],"centres":[23],"assessment":[24],"on":[25],"output":[27],"recommender":[30,46],"system":[31],"and,":[32],"therefore,":[33],"measures":[34],"performance":[35],"from":[36,127],"perspective":[38],"end-users.":[40],"However,":[41],"this":[42,64],"methodology":[43],"neglects":[44],"how":[45,173],"systems":[47],"more":[48],"broadly":[49],"model":[50],"user":[51,137,174],"preferences,":[52],"which":[53],"is":[54],"not":[55,132],"captured":[56],"by":[57,121,171],"only":[58],"considering":[59],"recommendations.":[62],"In":[63],"article,":[65],"we":[66,140,163],"item":[68,177],"response":[69],"theory":[70],"(IRT),":[71],"a":[72,84],"family":[73],"latent":[75,97,118],"variable":[76],"models":[77],"used":[78,109],"psychometric":[80],"assessment,":[81],"gain":[83],"comprehensive":[85],"understanding":[86],"evaluation.":[89],"We":[90],"IRT":[92,122],"jointly":[94],"estimate":[95],"abilities":[98,119],"51":[100],"algorithms":[102],"and":[103,176],"characteristics":[105],"3":[107],"commonly":[108],"benchmark":[110],"data":[111,115],"sets.":[112],"For":[113],"all":[114],"sets,":[116],"estimated":[120],"suggest":[123],"that":[124,142],"higher":[125],"scores":[126],"traditional":[128],"do":[131],"reflect":[133],"improvements":[134],"modeling":[136],"preferences.":[138],"Furthermore,":[139],"show":[141],"recommendations":[145],"with":[146],"most":[148],"discriminatory":[149],"power":[150],"are":[151],"biased":[152],"towards":[153],"lower":[154],"difficulty":[155],"items,":[156],"leaving":[157],"much":[158],"room":[159],"for":[160],"improvement.":[161],"Lastly,":[162],"highlight":[164],"role":[166],"popularity":[168,178],"investigating":[172],"engagement":[175],"influence":[179],"difficulty.":[181]},"counts_by_year":[{"year":2025,"cited_by_count":6}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}