{"id":"https://openalex.org/W4409158046","doi":"https://doi.org/10.1145/3690624.3709413","title":"Towards Web-scale Recommendations with LLMs: From Quality-aware Ranking to Candidate Generation","display_name":"Towards Web-scale Recommendations with LLMs: From Quality-aware Ranking to Candidate Generation","publication_year":2025,"publication_date":"2025-04-04","ids":{"openalex":"https://openalex.org/W4409158046","doi":"https://doi.org/10.1145/3690624.3709413"},"language":"en","primary_location":{"id":"doi:10.1145/3690624.3709413","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3690624.3709413","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 31st ACM SIGKDD Conference on Knowledge Discovery and Data Mining V.1","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3690624.3709413","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5066577149","display_name":"J.J. Shah","orcid":"https://orcid.org/0009-0004-7746-3000"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jaidev Shah","raw_affiliation_strings":["Microsoft AI, Redmond, WA, USA"],"raw_orcid":"https://orcid.org/0009-0004-7746-3000","affiliations":[{"raw_affiliation_string":"Microsoft AI, Redmond, WA, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5038200193","display_name":"Iman Barjasteh","orcid":null},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Iman Barjasteh","raw_affiliation_strings":["Microsoft AI, Redmond, WA, USA"],"raw_orcid":"https://orcid.org/0009-0004-4343-3444","affiliations":[{"raw_affiliation_string":"Microsoft AI, Redmond, WA, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5019652566","display_name":"Amey Barapatre","orcid":null},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Amey Barapatre","raw_affiliation_strings":["Microsoft AI, Redmond, WA, USA"],"raw_orcid":"https://orcid.org/0009-0007-3196-8136","affiliations":[{"raw_affiliation_string":"Microsoft AI, Redmond, WA, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5016701425","display_name":"Rana Forsati","orcid":null},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Rana Forsati","raw_affiliation_strings":["Microsoft AI, Redmond, WA, USA"],"raw_orcid":"https://orcid.org/0009-0002-7385-080X","affiliations":[{"raw_affiliation_string":"Microsoft AI, Redmond, WA, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5106687462","display_name":"Gang Luo","orcid":"https://orcid.org/0009-0009-3935-0562"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Gang Luo","raw_affiliation_strings":["Microsoft AI, Santa Clara, CA, USA"],"raw_orcid":"https://orcid.org/0009-0009-3935-0562","affiliations":[{"raw_affiliation_string":"Microsoft AI, Santa Clara, CA, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5011457751","display_name":"Fan Wu","orcid":"https://orcid.org/0009-0005-1252-6135"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Fan Wu","raw_affiliation_strings":["Microsoft AI, Redmond, WA, USA"],"raw_orcid":"https://orcid.org/0009-0005-1252-6135","affiliations":[{"raw_affiliation_string":"Microsoft AI, Redmond, WA, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Yuan Fang","orcid":"https://orcid.org/0009-0007-5838-7446"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yuan Fang","raw_affiliation_strings":["Microsoft AI, Redmond, WA, USA"],"raw_orcid":"https://orcid.org/0009-0007-5838-7446","affiliations":[{"raw_affiliation_string":"Microsoft AI, Redmond, WA, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5112471350","display_name":"Xue Deng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xue Deng","raw_affiliation_strings":["Microsoft AI, Suzhou, Suzhou, China"],"raw_orcid":"https://orcid.org/0009-0001-7246-8353","affiliations":[{"raw_affiliation_string":"Microsoft AI, Suzhou, Suzhou, China","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5038154162","display_name":"Blake Shepard","orcid":null},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Blake Shepard","raw_affiliation_strings":["Microsoft AI, Redmond, WA, USA"],"raw_orcid":"https://orcid.org/0009-0003-8376-753X","affiliations":[{"raw_affiliation_string":"Microsoft AI, Redmond, WA, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102446546","display_name":"Ronak Shah","orcid":null},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ronak Shah","raw_affiliation_strings":["Microsoft AI, Redmond, WA, USA"],"raw_orcid":"https://orcid.org/0009-0007-3181-9536","affiliations":[{"raw_affiliation_string":"Microsoft AI, Redmond, WA, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5036126537","display_name":"Linjun Yang","orcid":"https://orcid.org/0009-0001-1778-7167"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Linjun Yang","raw_affiliation_strings":["Microsoft AI, Redmond, WA, USA"],"raw_orcid":"https://orcid.org/0009-0001-1778-7167","affiliations":[{"raw_affiliation_string":"Microsoft AI, Redmond, WA, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5045605354","display_name":"Hongzhi Li","orcid":null},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Hongzhi Li","raw_affiliation_strings":["Microsoft AI, Redmond, WA, USA"],"raw_orcid":"https://orcid.org/0009-0003-4790-4349","affiliations":[{"raw_affiliation_string":"Microsoft AI, Redmond, WA, USA","institution_ids":["https://openalex.org/I1290206253"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"2514","last_page":"2524"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10215","display_name":"Semantic Web and Ontologies","score":0.9706000089645386,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10215","display_name":"Semantic Web and Ontologies","score":0.9706000089645386,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13523","display_name":"Mathematics, Computing, and Information Processing","score":0.9659000039100647,"subfield":{"id":"https://openalex.org/subfields/1703","display_name":"Computational Theory and Mathematics"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9599000215530396,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/ranking","display_name":"Ranking (information retrieval)","score":0.6507467031478882},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.629567563533783},{"id":"https://openalex.org/keywords/scale","display_name":"Scale (ratio)","score":0.5768446326255798},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.5264514088630676},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.3426308035850525},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.3270471692085266},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.30761876702308655},{"id":"https://openalex.org/keywords/geography","display_name":"Geography","score":0.09369602799415588}],"concepts":[{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.6507467031478882},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.629567563533783},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.5768446326255798},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.5264514088630676},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.3426308035850525},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.3270471692085266},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.30761876702308655},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.09369602799415588},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C58640448","wikidata":"https://www.wikidata.org/wiki/Q42515","display_name":"Cartography","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3690624.3709413","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3690624.3709413","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 31st ACM SIGKDD Conference on Knowledge Discovery and Data Mining V.1","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3690624.3709413","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3690624.3709413","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 31st ACM SIGKDD Conference on Knowledge Discovery and Data Mining V.1","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":20,"referenced_works":["https://openalex.org/W1511814458","https://openalex.org/W1522301498","https://openalex.org/W2343693914","https://openalex.org/W2990138404","https://openalex.org/W3008374555","https://openalex.org/W3030163527","https://openalex.org/W3035390927","https://openalex.org/W3171975879","https://openalex.org/W4221143046","https://openalex.org/W4297794619","https://openalex.org/W4308244910","https://openalex.org/W4310228395","https://openalex.org/W4367694290","https://openalex.org/W4376654514","https://openalex.org/W4385890179","https://openalex.org/W4386081793","https://openalex.org/W4392019445","https://openalex.org/W4396723112","https://openalex.org/W4400526908","https://openalex.org/W4402678047"],"related_works":["https://openalex.org/W2118564381","https://openalex.org/W2163901716","https://openalex.org/W2152204162","https://openalex.org/W2739821120","https://openalex.org/W2088097596","https://openalex.org/W2150136235","https://openalex.org/W2140661912","https://openalex.org/W2037724912","https://openalex.org/W2056806613","https://openalex.org/W2153069032"],"abstract_inverted_index":{"Explore":[0],"Further":[1],"@":[2],"Bing":[3,14],"is":[4],"a":[5,77,121],"webpage-to-webpage":[6],"recommendation":[7,43,54,95,109,125],"product,":[8],"enhancing":[9,40],"the":[10,22,47],"search":[11,23],"experience":[12],"on":[13],"by":[15,87],"surfacing":[16],"engaging":[17,94],"webpage":[18],"recommendations":[19],"tied":[20],"to":[21,66,90,118,143],"result":[24],"URLs.":[25],"In":[26],"this":[27],"paper,":[28],"we":[29,73,101,141],"present":[30],"our":[31,41,52,60,68,82,103,116,145,150],"approach":[32],"for":[33,39,63,105,149],"leveraging":[34,88],"Large":[35],"Language":[36],"Models":[37],"(LLMs)":[38],"web-scale":[42,151],"system.":[44],"We":[45,58,97,114,127],"describe":[46],"development":[48],"and":[49,93,132,139],"validation":[50],"of":[51],"LLM-powered":[53],"quality":[55,110],"metric":[56],"RecoDCG.":[57],"discuss":[59],"core":[61],"techniques":[62],"utilizing":[64],"LLMs":[65,89],"make":[67],"ranking":[69],"stage":[70,86],"quality-aware.":[71],"Furthermore,":[72],"detail":[74],"Q'":[75],"recall,":[76],"recall":[78],"path":[79],"that":[80],"enhances":[81],"system's":[83],"candidate":[84],"generation":[85],"produce":[91],"complementary":[92],"candidates.":[96],"also":[98],"address":[99],"how":[100],"optimize":[102],"system":[104],"multiple":[106],"objectives,":[107],"balancing":[108],"with":[111],"click":[112],"metrics.":[113],"deploy":[115],"work":[117],"production,":[119],"achieving":[120],"significant":[122],"improvement":[123],"in":[124],"quality.":[126],"share":[128],"results":[129],"from":[130],"offline":[131],"online":[133],"experiments":[134],"as":[135,137],"well":[136],"insights":[138],"steps":[140],"took":[142],"ensure":[144],"approaches":[146],"scale":[147],"effectively":[148],"needs.":[152]},"counts_by_year":[],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}