{"id":"https://openalex.org/W4409168760","doi":"https://doi.org/10.1145/3728373","title":"Recall, Robustness, and Lexicographic Evaluation","display_name":"Recall, Robustness, and Lexicographic Evaluation","publication_year":2025,"publication_date":"2025-04-04","ids":{"openalex":"https://openalex.org/W4409168760","doi":"https://doi.org/10.1145/3728373"},"language":"en","primary_location":{"id":"doi:10.1145/3728373","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3728373","pdf_url":null,"source":{"id":"https://openalex.org/S4387284325","display_name":"ACM Transactions on Recommender Systems","issn_l":"2770-6699","issn":["2770-6699"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Recommender Systems","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://doi.org/10.1145/3728373","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5101492251","display_name":"Fernando D\u00edaz","orcid":"https://orcid.org/0000-0003-2345-1288"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Fernando Diaz","raw_affiliation_strings":["Carnegie Mellon University","Carnegie Mellon University,  Pittsburgh, United States"],"raw_orcid":"https://orcid.org/0000-0003-2345-1288","affiliations":[{"raw_affiliation_string":"Carnegie Mellon University","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"Carnegie Mellon University,  Pittsburgh, United States","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5015092805","display_name":"Michael D. Ekstrand","orcid":"https://orcid.org/0000-0003-2467-0108"},"institutions":[{"id":"https://openalex.org/I72816309","display_name":"Drexel University","ror":"https://ror.org/04bdffz58","country_code":"US","type":"education","lineage":["https://openalex.org/I72816309"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Michael D. Ekstrand","raw_affiliation_strings":["Information Science, Drexel University","Information Science, Drexel University, Philadelphia, United States"],"raw_orcid":"https://orcid.org/0000-0003-2467-0108","affiliations":[{"raw_affiliation_string":"Information Science, Drexel University","institution_ids":["https://openalex.org/I72816309"]},{"raw_affiliation_string":"Information Science, Drexel University, Philadelphia, United States","institution_ids":["https://openalex.org/I72816309"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5048533217","display_name":"Bhaskar Mitra","orcid":"https://orcid.org/0000-0002-5270-5550"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]},{"id":"https://openalex.org/I4210105678","display_name":"Microsoft (Finland)","ror":"https://ror.org/01nehjf29","country_code":"FI","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210105678"]},{"id":"https://openalex.org/I4210153468","display_name":"Microsoft (Canada)","ror":"https://ror.org/04xhxg104","country_code":"CA","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210153468"]},{"id":"https://openalex.org/I4210164937","display_name":"Microsoft Research (United Kingdom)","ror":"https://ror.org/05k87vq12","country_code":"GB","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210164937"]}],"countries":["CA","FI","GB","US"],"is_corresponding":false,"raw_author_name":"Bhaskar Mitra","raw_affiliation_strings":["Research, Microsoft Corp","Research, Microsoft Corp, Montr\u00e9al, Canada"],"raw_orcid":"https://orcid.org/0000-0002-5270-5550","affiliations":[{"raw_affiliation_string":"Research, Microsoft Corp","institution_ids":["https://openalex.org/I1290206253","https://openalex.org/I4210105678","https://openalex.org/I4210164937"]},{"raw_affiliation_string":"Research, Microsoft Corp, Montr\u00e9al, Canada","institution_ids":["https://openalex.org/I4210153468"]}]}],"institutions":[],"countries_distinct_count":4,"institutions_distinct_count":6,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":6.98,"has_fulltext":false,"cited_by_count":6,"citation_normalized_percentile":{"value":0.96383513,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":98},"biblio":{"volume":"4","issue":"1","first_page":"1","last_page":"50"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9937000274658203,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9937000274658203,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9934999942779541,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10215","display_name":"Semantic Web and Ontologies","score":0.9925000071525574,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/lexicographical-order","display_name":"Lexicographical order","score":0.8024637699127197},{"id":"https://openalex.org/keywords/robustness","display_name":"Robustness (evolution)","score":0.7813400626182556},{"id":"https://openalex.org/keywords/recall","display_name":"Recall","score":0.6408307552337646},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.4908011853694916},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.35450422763824463},{"id":"https://openalex.org/keywords/cognitive-psychology","display_name":"Cognitive psychology","score":0.2539200782775879},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.2517567276954651},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.23717129230499268},{"id":"https://openalex.org/keywords/chemistry","display_name":"Chemistry","score":0.07509136199951172},{"id":"https://openalex.org/keywords/combinatorics","display_name":"Combinatorics","score":0.05461791157722473}],"concepts":[{"id":"https://openalex.org/C159254197","wikidata":"https://www.wikidata.org/wiki/Q1144915","display_name":"Lexicographical order","level":2,"score":0.8024637699127197},{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.7813400626182556},{"id":"https://openalex.org/C100660578","wikidata":"https://www.wikidata.org/wiki/Q18733","display_name":"Recall","level":2,"score":0.6408307552337646},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4908011853694916},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.35450422763824463},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.2539200782775879},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.2517567276954651},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.23717129230499268},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.07509136199951172},{"id":"https://openalex.org/C114614502","wikidata":"https://www.wikidata.org/wiki/Q76592","display_name":"Combinatorics","level":1,"score":0.05461791157722473},{"id":"https://openalex.org/C104317684","wikidata":"https://www.wikidata.org/wiki/Q7187","display_name":"Gene","level":2,"score":0.0},{"id":"https://openalex.org/C55493867","wikidata":"https://www.wikidata.org/wiki/Q7094","display_name":"Biochemistry","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3728373","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3728373","pdf_url":null,"source":{"id":"https://openalex.org/S4387284325","display_name":"ACM Transactions on Recommender Systems","issn_l":"2770-6699","issn":["2770-6699"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Recommender Systems","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1145/3728373","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3728373","pdf_url":null,"source":{"id":"https://openalex.org/S4387284325","display_name":"ACM Transactions on Recommender Systems","issn_l":"2770-6699","issn":["2770-6699"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Recommender Systems","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":117,"referenced_works":["https://openalex.org/W1420162645","https://openalex.org/W1521576632","https://openalex.org/W1577069963","https://openalex.org/W1578490632","https://openalex.org/W1966340639","https://openalex.org/W1968927634","https://openalex.org/W1969340322","https://openalex.org/W1971040550","https://openalex.org/W1974948938","https://openalex.org/W1982488265","https://openalex.org/W1989279326","https://openalex.org/W1989373893","https://openalex.org/W1990190154","https://openalex.org/W1994559819","https://openalex.org/W2001832483","https://openalex.org/W2004918107","https://openalex.org/W2005467652","https://openalex.org/W2011548984","https://openalex.org/W2017292914","https://openalex.org/W2027715414","https://openalex.org/W2031489019","https://openalex.org/W2032577510","https://openalex.org/W2034213747","https://openalex.org/W2037669632","https://openalex.org/W2040214686","https://openalex.org/W2046011778","https://openalex.org/W2047555304","https://openalex.org/W2048045485","https://openalex.org/W2051442094","https://openalex.org/W2053448995","https://openalex.org/W2059815359","https://openalex.org/W2061503185","https://openalex.org/W2063853488","https://openalex.org/W2070377439","https://openalex.org/W2071323102","https://openalex.org/W2074229683","https://openalex.org/W2075893676","https://openalex.org/W2104677329","https://openalex.org/W2107592605","https://openalex.org/W2109154214","https://openalex.org/W2109244020","https://openalex.org/W2116008435","https://openalex.org/W2130076000","https://openalex.org/W2137274315","https://openalex.org/W2141062075","https://openalex.org/W2142927605","https://openalex.org/W2160892561","https://openalex.org/W2164551773","https://openalex.org/W2168535456","https://openalex.org/W2168717408","https://openalex.org/W2231711798","https://openalex.org/W2250506749","https://openalex.org/W2263786666","https://openalex.org/W2288494888","https://openalex.org/W2309747864","https://openalex.org/W2341384651","https://openalex.org/W2560674852","https://openalex.org/W2612327332","https://openalex.org/W2740256306","https://openalex.org/W2742025582","https://openalex.org/W2786192177","https://openalex.org/W2787991113","https://openalex.org/W2795199972","https://openalex.org/W2798964384","https://openalex.org/W2798995557","https://openalex.org/W2945900074","https://openalex.org/W2963934714","https://openalex.org/W2994069334","https://openalex.org/W2995006168","https://openalex.org/W3002414756","https://openalex.org/W3002955717","https://openalex.org/W3017330897","https://openalex.org/W3023202929","https://openalex.org/W3028135017","https://openalex.org/W3034329167","https://openalex.org/W3035063214","https://openalex.org/W3035600984","https://openalex.org/W3040911419","https://openalex.org/W3089147479","https://openalex.org/W3094600186","https://openalex.org/W3099751307","https://openalex.org/W3101063193","https://openalex.org/W3101793783","https://openalex.org/W3101935024","https://openalex.org/W3102491798","https://openalex.org/W3102518922","https://openalex.org/W3128351871","https://openalex.org/W3134235469","https://openalex.org/W3156002164","https://openalex.org/W3163078977","https://openalex.org/W3167136514","https://openalex.org/W3185866970","https://openalex.org/W3193555562","https://openalex.org/W3197046229","https://openalex.org/W3201258060","https://openalex.org/W3208696530","https://openalex.org/W3215190495","https://openalex.org/W4214621108","https://openalex.org/W4224317071","https://openalex.org/W4226006895","https://openalex.org/W4229668452","https://openalex.org/W4233344962","https://openalex.org/W4241010992","https://openalex.org/W4251424354","https://openalex.org/W4251585144","https://openalex.org/W4283587264","https://openalex.org/W4284701291","https://openalex.org/W4285613665","https://openalex.org/W4296591825","https://openalex.org/W4296604448","https://openalex.org/W4300482433","https://openalex.org/W4312296047","https://openalex.org/W4386977429","https://openalex.org/W4389520307","https://openalex.org/W4392576636","https://openalex.org/W4401863302","https://openalex.org/W6634428267"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W1993634787","https://openalex.org/W3125614403","https://openalex.org/W4400317913","https://openalex.org/W2476923378","https://openalex.org/W3100240142","https://openalex.org/W2803221307","https://openalex.org/W4211001408"],"abstract_inverted_index":{"Although":[0],"originally":[1],"developed":[2],"to":[3,12,39,91,112,120,212],"evaluate":[4,13],"sets":[5],"of":[6,15,30,41,51,59,71,88,108,134,183,191,204],"items,":[7,16],"recall":[8,31,42,60,103,119,135,174,205],"is":[9,69,170],"often":[10],"used":[11],"rankings":[14,62],"including":[17],"those":[18],"produced":[19],"by":[20,136],"recommender,":[21],"retrieval,":[22],"and":[23,76,116,131,154,176,186,197,206,214],"other":[24],"machine":[25],"learning":[26],"systems.":[27],"The":[28],"application":[29],"without":[32],"a":[33,44,64,89,92,138],"formal":[34,65],"evaluative":[35],"motivation":[36],"has":[37,165],"led":[38],"criticism":[40],"as":[43,85],"vague":[45],"or":[46],"inappropriate":[47],"measure.":[48],"In":[49],"light":[50],"this":[52,129],"debate,":[53],"we":[54,80,101,127,157],"reflect":[55],"on":[56,144],"the":[57,86,106,189],"measurement":[58],"in":[61,95,181,188],"from":[63,105],"perspective.":[66],"Our":[67,194],"analysis":[68,150,199],"composed":[70],"three":[72],"tenets:":[73],"recall,":[74],"robustness,":[75],"lexicographic":[77,145],"evaluation.":[78],"First,":[79],"formally":[81],"define":[82],"\u201crecall":[83],"orientation\u201d":[84],"sensitivity":[87,180],"metric":[90],"user":[93],"interested":[94],"finding":[96],"every":[97],"relevant":[98],"item.":[99],"Second,":[100],"analyze":[102],"orientation":[104],"perspective":[107],"robustness":[109,213],"with":[110,172],"respect":[111],"possible":[113],"content":[114],"consumers":[115],"providers,":[117],"connecting":[118],"recent":[121],"conversations":[122],"about":[123],"fair":[124],"ranking.":[125],"Finally,":[126],"extend":[128],"conceptual":[130],"theoretical":[132],"treatment":[133],"developing":[137],"practical":[139],"preference-based":[140],"evaluation":[141,162],"method":[142],"based":[143],"comparison.":[146],"Through":[147],"extensive":[148],"empirical":[149,198],"across":[151],"multiple":[152],"recommendation":[153],"retrieval":[155],"tasks,":[156],"establish":[158],"that":[159],"our":[160,202],"new":[161],"method,":[163],"lexirecall,":[164],"convergent":[166],"validity":[167],"(i.e.,":[168],"it":[169],"correlated":[171],"existing":[173],"metrics)":[175],"exhibits":[177],"substantially":[178,200],"higher":[179],"terms":[182],"discriminative":[184],"power":[185],"stability":[187],"presence":[190],"missing":[192],"labels.":[193],"conceptual,":[195],"theoretical,":[196],"deepens":[201],"understanding":[203],"motivates":[207],"its":[208],"adoption":[209],"through":[210],"connections":[211],"fairness.":[215]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":4},{"year":2024,"cited_by_count":1}],"updated_date":"2026-05-21T06:26:12.895304","created_date":"2025-10-10T00:00:00"}