{"id":"https://openalex.org/W4412378001","doi":"https://doi.org/10.1145/3726302.3730310","title":"Investigating the Robustness of Counterfactual Learning to Rank Models: A Reproducibility Study","display_name":"Investigating the Robustness of Counterfactual Learning to Rank Models: A Reproducibility Study","publication_year":2025,"publication_date":"2025-07-13","ids":{"openalex":"https://openalex.org/W4412378001","doi":"https://doi.org/10.1145/3726302.3730310"},"language":"en","primary_location":{"id":"doi:10.1145/3726302.3730310","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3726302.3730310","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3726302.3730310","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 48th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3726302.3730310","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5102645619","display_name":"Zechun Niu","orcid":"https://orcid.org/0009-0009-7954-3713"},"institutions":[{"id":"https://openalex.org/I78988378","display_name":"Renmin University of China","ror":"https://ror.org/041pakw92","country_code":"CN","type":"education","lineage":["https://openalex.org/I78988378"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zechun Niu","raw_affiliation_strings":["Gaoling School of Artificial Intelligence, Renmin University of China, Beijing, China"],"raw_orcid":"https://orcid.org/0009-0009-7954-3713","affiliations":[{"raw_affiliation_string":"Gaoling School of Artificial Intelligence, Renmin University of China, Beijing, China","institution_ids":["https://openalex.org/I78988378"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Zhilin Zhang","orcid":"https://orcid.org/0009-0007-6713-3907"},"institutions":[{"id":"https://openalex.org/I78988378","display_name":"Renmin University of China","ror":"https://ror.org/041pakw92","country_code":"CN","type":"education","lineage":["https://openalex.org/I78988378"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhilin Zhang","raw_affiliation_strings":["School of Statistics, Renmin University of China, Beijing, China"],"raw_orcid":"https://orcid.org/0009-0007-6713-3907","affiliations":[{"raw_affiliation_string":"School of Statistics, Renmin University of China, Beijing, China","institution_ids":["https://openalex.org/I78988378"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5072119199","display_name":"Jiaxin Mao","orcid":"https://orcid.org/0000-0002-9257-5498"},"institutions":[{"id":"https://openalex.org/I78988378","display_name":"Renmin University of China","ror":"https://ror.org/041pakw92","country_code":"CN","type":"education","lineage":["https://openalex.org/I78988378"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jiaxin Mao","raw_affiliation_strings":["Gaoling School of Artificial Intelligence, Renmin University of China, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-9257-5498","affiliations":[{"raw_affiliation_string":"Gaoling School of Artificial Intelligence, Renmin University of China, Beijing, China","institution_ids":["https://openalex.org/I78988378"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5089655391","display_name":"Qingyao Ai","orcid":"https://orcid.org/0000-0002-5030-709X"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Qingyao Ai","raw_affiliation_strings":["Department of Computer Science and Technology, Tsinghua University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-5030-709X","affiliations":[{"raw_affiliation_string":"Department of Computer Science and Technology, Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5025631695","display_name":"Ji-Rong Wen","orcid":"https://orcid.org/0000-0002-9777-9676"},"institutions":[{"id":"https://openalex.org/I78988378","display_name":"Renmin University of China","ror":"https://ror.org/041pakw92","country_code":"CN","type":"education","lineage":["https://openalex.org/I78988378"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Ji-Rong Wen","raw_affiliation_strings":["Gaoling School of Artificial Intelligence, Renmin University of China, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-9777-9676","affiliations":[{"raw_affiliation_string":"Gaoling School of Artificial Intelligence, Renmin University of China, Beijing, China","institution_ids":["https://openalex.org/I78988378"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.0747867,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"3265","last_page":"3275"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9940000176429749,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9940000176429749,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.9825999736785889,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.9782999753952026,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/robustness","display_name":"Robustness (evolution)","score":0.8069175481796265},{"id":"https://openalex.org/keywords/counterfactual-thinking","display_name":"Counterfactual thinking","score":0.6806410551071167},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6317724585533142},{"id":"https://openalex.org/keywords/reproducibility","display_name":"Reproducibility","score":0.614570677280426},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.42028728127479553},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.39878663420677185},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.23903539776802063},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.21272435784339905},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.13284048438072205},{"id":"https://openalex.org/keywords/biology","display_name":"Biology","score":0.08097153902053833}],"concepts":[{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.8069175481796265},{"id":"https://openalex.org/C108650721","wikidata":"https://www.wikidata.org/wiki/Q1783253","display_name":"Counterfactual thinking","level":2,"score":0.6806410551071167},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6317724585533142},{"id":"https://openalex.org/C9893847","wikidata":"https://www.wikidata.org/wiki/Q1425625","display_name":"Reproducibility","level":2,"score":0.614570677280426},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.42028728127479553},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.39878663420677185},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.23903539776802063},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.21272435784339905},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.13284048438072205},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.08097153902053833},{"id":"https://openalex.org/C55493867","wikidata":"https://www.wikidata.org/wiki/Q7094","display_name":"Biochemistry","level":1,"score":0.0},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.0},{"id":"https://openalex.org/C104317684","wikidata":"https://www.wikidata.org/wiki/Q7187","display_name":"Gene","level":2,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3726302.3730310","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3726302.3730310","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3726302.3730310","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 48th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3726302.3730310","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3726302.3730310","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3726302.3730310","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 48th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},"sustainable_development_goals":[{"display_name":"Gender equality","id":"https://metadata.un.org/sdg/5","score":0.5199999809265137}],"awards":[{"id":"https://openalex.org/G6620384876","display_name":null,"funder_award_id":"62377044","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320322499","display_name":"Renmin University of China","ror":"https://ror.org/041pakw92"},{"id":"https://openalex.org/F4320335787","display_name":"Fundamental Research Funds for the Central Universities","ror":null}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4412378001.pdf","grobid_xml":"https://content.openalex.org/works/W4412378001.grobid-xml"},"referenced_works_count":52,"referenced_works":["https://openalex.org/W1974360117","https://openalex.org/W1982530130","https://openalex.org/W1992549066","https://openalex.org/W2003473029","https://openalex.org/W2009979684","https://openalex.org/W2026784708","https://openalex.org/W2035720976","https://openalex.org/W2049633694","https://openalex.org/W2069870183","https://openalex.org/W2090883204","https://openalex.org/W2099213975","https://openalex.org/W2113640060","https://openalex.org/W2150291618","https://openalex.org/W2152314154","https://openalex.org/W2154739689","https://openalex.org/W2155587858","https://openalex.org/W2279385734","https://openalex.org/W2339829457","https://openalex.org/W2340526403","https://openalex.org/W2507134384","https://openalex.org/W2535047536","https://openalex.org/W2536015822","https://openalex.org/W2539671052","https://openalex.org/W2769473018","https://openalex.org/W2797400361","https://openalex.org/W2798492560","https://openalex.org/W2798634418","https://openalex.org/W2799226155","https://openalex.org/W2890291106","https://openalex.org/W2948795993","https://openalex.org/W2955421345","https://openalex.org/W2997919341","https://openalex.org/W3012600133","https://openalex.org/W3026200234","https://openalex.org/W3028135017","https://openalex.org/W3099404779","https://openalex.org/W3101148092","https://openalex.org/W3101935024","https://openalex.org/W3103733168","https://openalex.org/W3105712174","https://openalex.org/W3115087172","https://openalex.org/W3147653952","https://openalex.org/W3153783849","https://openalex.org/W3155345376","https://openalex.org/W4283654062","https://openalex.org/W4288079518","https://openalex.org/W4367860606","https://openalex.org/W4393968591","https://openalex.org/W4400524681","https://openalex.org/W4400524979","https://openalex.org/W4401202336","https://openalex.org/W6639094827"],"related_works":["https://openalex.org/W2961085424","https://openalex.org/W4306674287","https://openalex.org/W4387369504","https://openalex.org/W4394896187","https://openalex.org/W3170094116","https://openalex.org/W4386462264","https://openalex.org/W3107602296","https://openalex.org/W4364306694","https://openalex.org/W4312192474","https://openalex.org/W4283697347"],"abstract_inverted_index":{"Counterfactual":[0],"learning":[1],"to":[2,16,23,58,93,106],"rank":[3],"(CLTR)":[4],"has":[5],"attracted":[6],"extensive":[7],"attention":[8],"in":[9,128],"the":[10,28,36,43,84,123],"IR":[11],"community":[12],"for":[13],"its":[14],"ability":[15],"leverage":[17],"massive":[18],"logged":[19],"user":[20,37,103,108],"interaction":[21],"data":[22],"train":[24],"ranking":[25],"models.":[26],"While":[27],"CLTR":[29,126],"models":[30,127],"can":[31],"be":[32],"theoretically":[33],"unbiased":[34],"when":[35],"behavior":[38],"assumption":[39],"is":[40,46,50,133],"correct":[41],"and":[42,110,130,136],"propensity":[44],"estimation":[45],"accurate,":[47],"their":[48],"effectiveness":[49],"usually":[51],"empirically":[52],"evaluated":[53],"via":[54],"simulation-based":[55,71],"experiments":[56,72],"due":[57],"a":[59,89,101,113,121],"lack":[60],"of":[61,83,116,125],"widely":[62],"available,":[63],"large-scale,":[64],"real":[65],"click":[66,118],"logs.":[67,119],"However,":[68],"many":[69],"previous":[70],"are":[73],"somewhat":[74],"limited":[75],"because":[76],"they":[77],"may":[78],"have":[79],"one":[80],"or":[81],"more":[82],"following":[85],"deficiencies:":[86],"1)":[87],"using":[88],"weak":[90],"production":[91],"ranker":[92],"generate":[94],"initial":[95],"ranked":[96],"lists,":[97],"2)":[98],"relying":[99],"on":[100],"simplified":[102],"simulation":[104],"model":[105],"simulate":[107],"clicks,":[109],"3)":[111],"generating":[112],"fixed":[114],"number":[115],"synthetic":[117],"As":[120],"result,":[122],"robustness":[124],"complex":[129],"diverse":[131],"situations":[132],"largely":[134],"unknown":[135],"needs":[137],"further":[138],"investigation.":[139]},"counts_by_year":[],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-10-10T00:00:00"}