{"id":"https://openalex.org/W4412876913","doi":"https://doi.org/10.1145/3711896.3737232","title":"Harnessing the Power of Interleaving and Counterfactual Evaluation for Airbnb Search Ranking","display_name":"Harnessing the Power of Interleaving and Counterfactual Evaluation for Airbnb Search Ranking","publication_year":2025,"publication_date":"2025-08-03","ids":{"openalex":"https://openalex.org/W4412876913","doi":"https://doi.org/10.1145/3711896.3737232"},"language":"en","primary_location":{"id":"doi:10.1145/3711896.3737232","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3711896.3737232","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3711896.3737232","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 31st ACM SIGKDD Conference on Knowledge Discovery and Data Mining V.2","raw_type":"proceedings-article"},"type":"preprint","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3711896.3737232","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5035621099","display_name":"Qing Zhang","orcid":"https://orcid.org/0009-0005-3572-2606"},"institutions":[{"id":"https://openalex.org/I106110158","display_name":"Bay Area Air Quality Management District","ror":"https://ror.org/04431t173","country_code":"US","type":"government","lineage":["https://openalex.org/I106110158"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Qing Zhang","raw_affiliation_strings":["Airbnb, San Francisco, CA, USA"],"raw_orcid":"https://orcid.org/0009-0005-3572-2606","affiliations":[{"raw_affiliation_string":"Airbnb, San Francisco, CA, USA","institution_ids":["https://openalex.org/I106110158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5075519075","display_name":"Alex Deng","orcid":"https://orcid.org/0000-0002-8116-5602"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Alex Deng","raw_affiliation_strings":["Microsoft, Seattle, WA, USA"],"raw_orcid":"https://orcid.org/0000-0002-8116-5602","affiliations":[{"raw_affiliation_string":"Microsoft, Seattle, WA, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5048623684","display_name":"Michelle Du","orcid":"https://orcid.org/0009-0006-0142-6718"},"institutions":[{"id":"https://openalex.org/I106110158","display_name":"Bay Area Air Quality Management District","ror":"https://ror.org/04431t173","country_code":"US","type":"government","lineage":["https://openalex.org/I106110158"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Michelle Du","raw_affiliation_strings":["Airbnb, San Francisco, CA, USA"],"raw_orcid":"https://orcid.org/0009-0006-0142-6718","affiliations":[{"raw_affiliation_string":"Airbnb, San Francisco, CA, USA","institution_ids":["https://openalex.org/I106110158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5068625546","display_name":"Huiji Gao","orcid":"https://orcid.org/0009-0006-0424-248X"},"institutions":[{"id":"https://openalex.org/I106110158","display_name":"Bay Area Air Quality Management District","ror":"https://ror.org/04431t173","country_code":"US","type":"government","lineage":["https://openalex.org/I106110158"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Huiji Gao","raw_affiliation_strings":["Airbnb, San Francisco, CA, USA"],"raw_orcid":"https://orcid.org/0009-0006-0424-248X","affiliations":[{"raw_affiliation_string":"Airbnb, San Francisco, CA, USA","institution_ids":["https://openalex.org/I106110158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5092061331","display_name":"Liwei He","orcid":"https://orcid.org/0009-0007-2942-3145"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liwei He","raw_affiliation_strings":["Airbnb, Seattle, WA, USA"],"raw_orcid":"https://orcid.org/0009-0007-2942-3145","affiliations":[{"raw_affiliation_string":"Airbnb, Seattle, WA, USA","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5087282245","display_name":"Sanjeev Katariya","orcid":"https://orcid.org/0009-0008-1519-174X"},"institutions":[{"id":"https://openalex.org/I106110158","display_name":"Bay Area Air Quality Management District","ror":"https://ror.org/04431t173","country_code":"US","type":"government","lineage":["https://openalex.org/I106110158"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Sanjeev Katariya","raw_affiliation_strings":["Airbnb, San Francisco, CA, USA"],"raw_orcid":"https://orcid.org/0009-0008-1519-174X","affiliations":[{"raw_affiliation_string":"Airbnb, San Francisco, CA, USA","institution_ids":["https://openalex.org/I106110158"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":6,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.26128673,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"5205","last_page":"5214"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10203","display_name":"Recommender Systems and Techniques","score":0.9987000226974487,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10203","display_name":"Recommender Systems and Techniques","score":0.9987000226974487,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12392","display_name":"Sharing Economy and Platforms","score":0.9925000071525574,"subfield":{"id":"https://openalex.org/subfields/1406","display_name":"Marketing"},"field":{"id":"https://openalex.org/fields/14","display_name":"Business, Management and Accounting"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11644","display_name":"Spam and Phishing Detection","score":0.9904000163078308,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/interleaving","display_name":"Interleaving","score":0.9002031683921814},{"id":"https://openalex.org/keywords/counterfactual-thinking","display_name":"Counterfactual thinking","score":0.8054872751235962},{"id":"https://openalex.org/keywords/ranking","display_name":"Ranking (information retrieval)","score":0.6163467168807983},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5793697237968445},{"id":"https://openalex.org/keywords/power","display_name":"Power (physics)","score":0.45683926343917847},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.33880674839019775},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.1109657883644104}],"concepts":[{"id":"https://openalex.org/C28034677","wikidata":"https://www.wikidata.org/wiki/Q17092530","display_name":"Interleaving","level":2,"score":0.9002031683921814},{"id":"https://openalex.org/C108650721","wikidata":"https://www.wikidata.org/wiki/Q1783253","display_name":"Counterfactual thinking","level":2,"score":0.8054872751235962},{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.6163467168807983},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5793697237968445},{"id":"https://openalex.org/C163258240","wikidata":"https://www.wikidata.org/wiki/Q25342","display_name":"Power (physics)","level":2,"score":0.45683926343917847},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.33880674839019775},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.1109657883644104},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3711896.3737232","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3711896.3737232","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3711896.3737232","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 31st ACM SIGKDD Conference on Knowledge Discovery and Data Mining V.2","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2508.00751","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2508.00751","pdf_url":"https://arxiv.org/pdf/2508.00751","source":{"id":"https://openalex.org/S4393918464","display_name":"ArXiv.org","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"}],"best_oa_location":{"id":"doi:10.1145/3711896.3737232","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3711896.3737232","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3711896.3737232","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 31st ACM SIGKDD Conference on Knowledge Discovery and Data Mining V.2","raw_type":"proceedings-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/13","display_name":"Climate action","score":0.75}],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4412876913.pdf","grobid_xml":"https://content.openalex.org/works/W4412876913.grobid-xml"},"referenced_works_count":25,"referenced_works":["https://openalex.org/W1982530130","https://openalex.org/W2047221353","https://openalex.org/W2069870183","https://openalex.org/W2073248814","https://openalex.org/W2094790959","https://openalex.org/W2126002144","https://openalex.org/W2150291618","https://openalex.org/W2156541733","https://openalex.org/W2163996478","https://openalex.org/W2463677609","https://openalex.org/W2509295096","https://openalex.org/W2584778415","https://openalex.org/W2784068709","https://openalex.org/W2902572901","https://openalex.org/W3003609932","https://openalex.org/W3041933364","https://openalex.org/W3099420497","https://openalex.org/W3150893739","https://openalex.org/W4233471163","https://openalex.org/W4306317018","https://openalex.org/W4378942346","https://openalex.org/W4385562609","https://openalex.org/W4401863347","https://openalex.org/W4401863592","https://openalex.org/W4414435001"],"related_works":["https://openalex.org/W2118564381","https://openalex.org/W2163901716","https://openalex.org/W2152204162","https://openalex.org/W2739821120","https://openalex.org/W2088097596","https://openalex.org/W2150136235","https://openalex.org/W2140661912","https://openalex.org/W2037724912","https://openalex.org/W2056806613","https://openalex.org/W2153069032"],"abstract_inverted_index":{"Evaluation":[0],"plays":[1],"a":[2,28,147],"crucial":[3],"role":[4],"in":[5,27,59,175],"the":[6,131,142,155,167],"development":[7],"of":[8,144,149,187],"ranking":[9],"algorithms":[10],"on":[11,154],"search":[12],"and":[13,30,63,98,105,120,157,191],"recommender":[14],"systems.It":[15],"enables":[16],"online":[17,33,127],"platforms":[18],"to":[19,38,57,73,124,151,160],"create":[20],"user-friendly":[21],"features":[22],"that":[23],"drive":[24],"commercial":[25],"success":[26],"steady":[29],"effective":[31,74],"manner.The":[32],"environment":[34],"is":[35],"particularly":[36],"conducive":[37],"applying":[39],"causal":[40],"inference":[41],"techniques,":[42],"such":[43],"as":[44,49],"randomized":[45],"controlled":[46],"experiments":[47,145],"(known":[48],"A/B":[50,75,112,136,162],"test),":[51],"which":[52],"are":[53,96,106],"often":[54,102],"more":[55,99],"challenging":[56],"implement":[58],"fields":[60],"like":[61,91],"medicine":[62],"public":[64],"policy.However,":[65],"businesses":[66],"face":[67],"unique":[68],"challenges":[69],"when":[70],"it":[71],"comes":[72],"test.Specifically,":[76],"achieving":[77],"sufficient":[78],"statistical":[79],"power":[80],"for":[81,88,108,111,129,135],"conversion-based":[82],"metrics":[83],"can":[84,177],"be":[85],"time-consuming,":[86],"especially":[87],"significant":[89],"purchases":[90],"booking":[92],"accommodations.While":[93],"offline":[94],"evaluations":[95],"quicker":[97],"cost-effective,":[100],"they":[101],"lack":[103],"accuracy":[104],"inadequate":[107],"selecting":[109],"candidates":[110,134],"test.To":[113],"address":[114],"these":[115],"challenges,":[116],"we":[117],"developed":[118],"interleaving":[119],"counterfactual":[121],"evaluation":[122],"methods":[123],"facilitate":[125],"rapid":[126],"assessments":[128],"identifying":[130],"most":[132],"promising":[133],"tests.Our":[137],"approach":[138,156],"not":[139],"only":[140],"increased":[141],"sensitivity":[143],"by":[146],"factor":[148],"up":[150],"100":[152],"(depending":[153],"metrics)":[158],"compared":[159],"traditional":[161],"testing":[163,190],"but":[164],"also":[165,178],"streamlined":[166],"experimental":[168],"process.The":[169],"practical":[170],"insights":[171],"gained":[172],"from":[173],"usage":[174],"production":[176],"benefit":[179],"organizations":[180],"with":[181],"similar":[182],"interests.":[183],"CCS":[184],"Concepts":[185],"Mathematics":[186],"computing":[188,196],"Hypothesis":[189],"confidence":[192],"interval":[193],"computation;":[194],"Applied":[195],"Electronic":[197],"commerce.":[198]},"counts_by_year":[],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-10-10T00:00:00"}