{"id":"https://openalex.org/W3152757293","doi":"https://doi.org/10.1145/3442381.3450080","title":"Estimation of Fair Ranking Metrics with Incomplete Judgments","display_name":"Estimation of Fair Ranking Metrics with Incomplete Judgments","publication_year":2021,"publication_date":"2021-04-19","ids":{"openalex":"https://openalex.org/W3152757293","doi":"https://doi.org/10.1145/3442381.3450080","mag":"3152757293"},"language":"en","primary_location":{"id":"doi:10.1145/3442381.3450080","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3442381.3450080","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Web Conference 2021","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3442381.3450080","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5078393320","display_name":"\u00d6mer K\u0131rnap","orcid":null},"institutions":[{"id":"https://openalex.org/I45129253","display_name":"University College London","ror":"https://ror.org/02jx3x895","country_code":"GB","type":"education","lineage":["https://openalex.org/I124357947","https://openalex.org/I45129253"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"\u00d6mer K\u0131rnap","raw_affiliation_strings":["University College London, United Kingdom"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University College London, United Kingdom","institution_ids":["https://openalex.org/I45129253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101492251","display_name":"Fernando D\u00edaz","orcid":"https://orcid.org/0000-0003-2345-1288"},"institutions":[{"id":"https://openalex.org/I4210153468","display_name":"Microsoft (Canada)","ror":"https://ror.org/04xhxg104","country_code":"CA","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210153468"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Fernando Diaz","raw_affiliation_strings":["Microsoft Research, Canada"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Research, Canada","institution_ids":["https://openalex.org/I4210153468"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5087294637","display_name":"Asia J. Biega","orcid":"https://orcid.org/0000-0001-8083-0976"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Asia Biega","raw_affiliation_strings":["Max Planck Institute for Security and Privacy, Canada"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Max Planck Institute for Security and Privacy, Canada","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5015092805","display_name":"Michael D. Ekstrand","orcid":"https://orcid.org/0000-0003-2467-0108"},"institutions":[{"id":"https://openalex.org/I120156002","display_name":"Boise State University","ror":"https://ror.org/02e3zdp86","country_code":"US","type":"education","lineage":["https://openalex.org/I120156002"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Michael Ekstrand","raw_affiliation_strings":["Boise State University, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Boise State University, USA","institution_ids":["https://openalex.org/I120156002"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5078123101","display_name":"Ben Carterette","orcid":"https://orcid.org/0000-0001-9538-047X"},"institutions":[{"id":"https://openalex.org/I4210122154","display_name":"Photon Spot (United States)","ror":"https://ror.org/01yxc0v75","country_code":"US","type":"company","lineage":["https://openalex.org/I4210122154"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ben Carterette","raw_affiliation_strings":["Spotify Research, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Spotify Research, USA","institution_ids":["https://openalex.org/I4210122154"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5076265623","display_name":"Emine Y\u0131lmaz","orcid":"https://orcid.org/0000-0002-3434-8932"},"institutions":[{"id":"https://openalex.org/I45129253","display_name":"University College London","ror":"https://ror.org/02jx3x895","country_code":"GB","type":"education","lineage":["https://openalex.org/I124357947","https://openalex.org/I45129253"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Emine Yilmaz","raw_affiliation_strings":["University College London, United Kingdom"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University College London, United Kingdom","institution_ids":["https://openalex.org/I45129253"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":40,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1065","last_page":"1075"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11182","display_name":"Auction Theory and Applications","score":0.9972000122070312,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T11182","display_name":"Auction Theory and Applications","score":0.9972000122070312,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10991","display_name":"Game Theory and Voting Systems","score":0.9932000041007996,"subfield":{"id":"https://openalex.org/subfields/2002","display_name":"Economics and Econometrics"},"field":{"id":"https://openalex.org/fields/20","display_name":"Economics, Econometrics and Finance"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9847000241279602,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/ranking","display_name":"Ranking (information retrieval)","score":0.8236173391342163},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6546696424484253},{"id":"https://openalex.org/keywords/estimator","display_name":"Estimator","score":0.6225778460502625},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.5286598205566406},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.4405856132507324},{"id":"https://openalex.org/keywords/completeness","display_name":"Completeness (order theory)","score":0.4135918915271759},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.2915239930152893},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.21691036224365234}],"concepts":[{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.8236173391342163},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6546696424484253},{"id":"https://openalex.org/C185429906","wikidata":"https://www.wikidata.org/wiki/Q1130160","display_name":"Estimator","level":2,"score":0.6225778460502625},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.5286598205566406},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4405856132507324},{"id":"https://openalex.org/C17231256","wikidata":"https://www.wikidata.org/wiki/Q5156540","display_name":"Completeness (order theory)","level":2,"score":0.4135918915271759},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.2915239930152893},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.21691036224365234},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3442381.3450080","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3442381.3450080","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Web Conference 2021","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3442381.3450080","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3442381.3450080","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Web Conference 2021","raw_type":"proceedings-article"},"sustainable_development_goals":[{"score":0.7799999713897705,"display_name":"Gender equality","id":"https://metadata.un.org/sdg/5"}],"awards":[{"id":"https://openalex.org/G7929798828","display_name":null,"funder_award_id":"EP/P024289/1","funder_id":"https://openalex.org/F4320334627","funder_display_name":"Engineering and Physical Sciences Research Council"}],"funders":[{"id":"https://openalex.org/F4320334627","display_name":"Engineering and Physical Sciences Research Council","ror":"https://ror.org/0439y7842"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":38,"referenced_works":["https://openalex.org/W32360141","https://openalex.org/W1593806347","https://openalex.org/W1967879792","https://openalex.org/W1968927634","https://openalex.org/W1992549066","https://openalex.org/W2009954908","https://openalex.org/W2048426792","https://openalex.org/W2057744254","https://openalex.org/W2069870183","https://openalex.org/W2100960835","https://openalex.org/W2113640060","https://openalex.org/W2115536324","https://openalex.org/W2122946987","https://openalex.org/W2124504084","https://openalex.org/W2132314908","https://openalex.org/W2402441596","https://openalex.org/W2544318541","https://openalex.org/W2704480242","https://openalex.org/W2787991113","https://openalex.org/W2806229618","https://openalex.org/W2893359107","https://openalex.org/W2894728481","https://openalex.org/W2903875172","https://openalex.org/W2913539872","https://openalex.org/W2963189767","https://openalex.org/W2990592470","https://openalex.org/W3003137157","https://openalex.org/W3013218228","https://openalex.org/W3023202929","https://openalex.org/W3028722847","https://openalex.org/W3094600186","https://openalex.org/W3099361686","https://openalex.org/W3102092462","https://openalex.org/W3102518922","https://openalex.org/W3105035347","https://openalex.org/W3128736136","https://openalex.org/W4288359825","https://openalex.org/W7047218413"],"related_works":["https://openalex.org/W4237277701","https://openalex.org/W4287880334","https://openalex.org/W2121424666","https://openalex.org/W1587028174","https://openalex.org/W4301250310","https://openalex.org/W2030928163","https://openalex.org/W4366700029","https://openalex.org/W4285230481","https://openalex.org/W2067700950","https://openalex.org/W1669998927"],"abstract_inverted_index":{"There":[0],"is":[1],"increasing":[2],"attention":[3],"to":[4,21,72,139],"evaluating":[5],"the":[6,17,40,51,60],"fairness":[7],"of":[8,19,44,48,54,62,104,129],"search":[9],"system":[10],"ranking":[11,64,87,131],"decisions.":[12],"These":[13],"metrics":[14,37,65,132],"often":[15,24],"consider":[16],"membership":[18],"items":[20],"particular":[22],"groups,":[23],"identified":[25],"using":[26,111],"protected":[27,45,52],"attributes":[28,53],"such":[29],"as":[30],"gender":[31],"or":[32,141],"ethnicity.":[33],"To":[34],"date,":[35],"these":[36],"typically":[38],"assume":[39],"availability":[41],"and":[42,81,93,114,133],"completeness":[43],"attribute":[46],"labels":[47],"items.":[49,106],"However,":[50],"individuals":[55],"are":[56],"rarely":[57],"present,":[58],"limiting":[59],"application":[61],"fair":[63,86,130],"in":[66],"large":[67],"scale":[68],"systems.":[69],"In":[70],"order":[71],"address":[73],"this":[74,127],"problem,":[75],"we":[76],"propose":[77],"a":[78,91,135],"sampling":[79],"strategy":[80],"estimation":[82],"technique":[83],"for":[84],"four":[85],"metrics.":[88],"We":[89,107],"formulate":[90],"robust":[92],"unbiased":[94],"estimator":[95],"which":[96],"can":[97,125],"operate":[98],"even":[99],"with":[100],"very":[101],"limited":[102],"number":[103],"labeled":[105],"evaluate":[108],"our":[109,123],"approach":[110],"both":[112],"simulated":[113],"real":[115],"world":[116],"data.":[117],"Our":[118],"experimental":[119],"results":[120],"demonstrate":[121],"that":[122],"method":[124],"estimate":[126],"family":[128],"provides":[134],"robust,":[136],"reliable":[137],"alternative":[138],"exhaustive":[140],"random":[142],"data":[143],"annotation.":[144]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":14},{"year":2023,"cited_by_count":10},{"year":2022,"cited_by_count":11},{"year":2021,"cited_by_count":1}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}