{"id":"https://openalex.org/W3153557349","doi":"https://doi.org/10.1145/3404835.3462951","title":"Not All Relevance Scores are Equal: Efficient Uncertainty and Calibration Modeling for Deep Retrieval Models","display_name":"Not All Relevance Scores are Equal: Efficient Uncertainty and Calibration Modeling for Deep Retrieval Models","publication_year":2021,"publication_date":"2021-07-11","ids":{"openalex":"https://openalex.org/W3153557349","doi":"https://doi.org/10.1145/3404835.3462951","mag":"3153557349"},"language":"en","primary_location":{"id":"doi:10.1145/3404835.3462951","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3404835.3462951","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3404835.3462951","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 44th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3404835.3462951","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Daniel Cohen","orcid":null},"institutions":[{"id":"https://openalex.org/I27804330","display_name":"Brown University","ror":"https://ror.org/05gq02987","country_code":"US","type":"education","lineage":["https://openalex.org/I27804330"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Daniel Cohen","raw_affiliation_strings":["Brown University, Providence, RI, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Brown University, Providence, RI, USA","institution_ids":["https://openalex.org/I27804330"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Bhaskar Mitra","orcid":null},"institutions":[{"id":"https://openalex.org/I4210153468","display_name":"Microsoft (Canada)","ror":"https://ror.org/04xhxg104","country_code":"CA","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210153468"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Bhaskar Mitra","raw_affiliation_strings":["Microsoft, Montreal, PQ, Canada"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft, Montreal, PQ, Canada","institution_ids":["https://openalex.org/I4210153468"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Oleg Lesota","orcid":null},"institutions":[{"id":"https://openalex.org/I121883995","display_name":"Johannes Kepler University of Linz","ror":"https://ror.org/052r2xn60","country_code":"AT","type":"education","lineage":["https://openalex.org/I121883995"]}],"countries":["AT"],"is_corresponding":false,"raw_author_name":"Oleg Lesota","raw_affiliation_strings":["Johannes Kepler University, Linz, Austria"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Johannes Kepler University, Linz, Austria","institution_ids":["https://openalex.org/I121883995"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Navid Rekabsaz","orcid":null},"institutions":[{"id":"https://openalex.org/I121883995","display_name":"Johannes Kepler University of Linz","ror":"https://ror.org/052r2xn60","country_code":"AT","type":"education","lineage":["https://openalex.org/I121883995"]}],"countries":["AT"],"is_corresponding":false,"raw_author_name":"Navid Rekabsaz","raw_affiliation_strings":["Johannes Kepler University, Linz, Austria"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Johannes Kepler University, Linz, Austria","institution_ids":["https://openalex.org/I121883995"]}]},{"author_position":"last","author":{"id":null,"display_name":"Carsten Eickhoff","orcid":null},"institutions":[{"id":"https://openalex.org/I27804330","display_name":"Brown University","ror":"https://ror.org/05gq02987","country_code":"US","type":"education","lineage":["https://openalex.org/I27804330"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Carsten Eickhoff","raw_affiliation_strings":["Brown University, Providence, RI, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Brown University, Providence, RI, USA","institution_ids":["https://openalex.org/I27804330"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":18,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"654","last_page":"664"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.998199999332428,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.9980999827384949,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/ranking","display_name":"Ranking (information retrieval)","score":0.7925999760627747},{"id":"https://openalex.org/keywords/relevance","display_name":"Relevance (law)","score":0.7804999947547913},{"id":"https://openalex.org/keywords/metric","display_name":"Metric (unit)","score":0.5957000255584717},{"id":"https://openalex.org/keywords/calibration","display_name":"Calibration","score":0.5012999773025513},{"id":"https://openalex.org/keywords/bayesian-probability","display_name":"Bayesian probability","score":0.4894999861717224},{"id":"https://openalex.org/keywords/divergence-from-randomness-model","display_name":"Divergence-from-randomness model","score":0.4787999987602234},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.3935000002384186},{"id":"https://openalex.org/keywords/scope","display_name":"Scope (computer science)","score":0.38960000872612},{"id":"https://openalex.org/keywords/bayesian-inference","display_name":"Bayesian inference","score":0.3871000111103058}],"concepts":[{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.7925999760627747},{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.7804999947547913},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7085000276565552},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.5957000255584717},{"id":"https://openalex.org/C165838908","wikidata":"https://www.wikidata.org/wiki/Q736777","display_name":"Calibration","level":2,"score":0.5012999773025513},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.49470001459121704},{"id":"https://openalex.org/C107673813","wikidata":"https://www.wikidata.org/wiki/Q812534","display_name":"Bayesian probability","level":2,"score":0.4894999861717224},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.48899999260902405},{"id":"https://openalex.org/C149189445","wikidata":"https://www.wikidata.org/wiki/Q5283894","display_name":"Divergence-from-randomness model","level":3,"score":0.4787999987602234},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4715999960899353},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.3935000002384186},{"id":"https://openalex.org/C2778012447","wikidata":"https://www.wikidata.org/wiki/Q1034415","display_name":"Scope (computer science)","level":2,"score":0.38960000872612},{"id":"https://openalex.org/C160234255","wikidata":"https://www.wikidata.org/wiki/Q812535","display_name":"Bayesian inference","level":3,"score":0.3871000111103058},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3779999911785126},{"id":"https://openalex.org/C12725497","wikidata":"https://www.wikidata.org/wiki/Q810247","display_name":"Baseline (sea)","level":2,"score":0.36390000581741333},{"id":"https://openalex.org/C161156560","wikidata":"https://www.wikidata.org/wiki/Q1638872","display_name":"Document retrieval","level":2,"score":0.32690000534057617},{"id":"https://openalex.org/C99016210","wikidata":"https://www.wikidata.org/wiki/Q5488129","display_name":"Query expansion","level":2,"score":0.32019999623298645},{"id":"https://openalex.org/C86037889","wikidata":"https://www.wikidata.org/wiki/Q4330127","display_name":"Learning to rank","level":3,"score":0.3174000084400177},{"id":"https://openalex.org/C2779532271","wikidata":"https://www.wikidata.org/wiki/Q445558","display_name":"Relevance feedback","level":4,"score":0.3122999966144562},{"id":"https://openalex.org/C89686163","wikidata":"https://www.wikidata.org/wiki/Q1187982","display_name":"Vector space model","level":2,"score":0.3046000003814697},{"id":"https://openalex.org/C22639730","wikidata":"https://www.wikidata.org/wiki/Q7702546","display_name":"Term Discrimination","level":5,"score":0.29739999771118164},{"id":"https://openalex.org/C44291984","wikidata":"https://www.wikidata.org/wiki/Q1074173","display_name":"Question answering","level":2,"score":0.296999990940094},{"id":"https://openalex.org/C49937458","wikidata":"https://www.wikidata.org/wiki/Q2599292","display_name":"Probabilistic logic","level":2,"score":0.2838999927043915},{"id":"https://openalex.org/C33724603","wikidata":"https://www.wikidata.org/wiki/Q812540","display_name":"Bayesian network","level":2,"score":0.2815999984741211},{"id":"https://openalex.org/C61797465","wikidata":"https://www.wikidata.org/wiki/Q1188986","display_name":"Term (time)","level":2,"score":0.28060001134872437},{"id":"https://openalex.org/C2780898871","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Performance metric","level":2,"score":0.27709999680519104},{"id":"https://openalex.org/C90288658","wikidata":"https://www.wikidata.org/wiki/Q3318149","display_name":"Human\u2013computer information retrieval","level":3,"score":0.2766999900341034}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3404835.3462951","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3404835.3462951","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3404835.3462951","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 44th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2105.04651","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2105.04651","pdf_url":"https://arxiv.org/pdf/2105.04651","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"}],"best_oa_location":{"id":"doi:10.1145/3404835.3462951","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3404835.3462951","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3404835.3462951","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 44th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3153557349.pdf","grobid_xml":"https://content.openalex.org/works/W3153557349.grobid-xml"},"referenced_works_count":26,"referenced_works":["https://openalex.org/W1647779468","https://openalex.org/W2051025610","https://openalex.org/W2079370602","https://openalex.org/W2087131461","https://openalex.org/W2095705004","https://openalex.org/W2132314908","https://openalex.org/W2469819362","https://openalex.org/W2605183330","https://openalex.org/W2758859575","https://openalex.org/W2759472822","https://openalex.org/W2783640434","https://openalex.org/W2809897079","https://openalex.org/W2945127593","https://openalex.org/W2962992635","https://openalex.org/W2970419734","https://openalex.org/W2976416215","https://openalex.org/W2976611402","https://openalex.org/W3021397474","https://openalex.org/W3023202929","https://openalex.org/W3034580237","https://openalex.org/W3035282196","https://openalex.org/W3043859333","https://openalex.org/W3044284384","https://openalex.org/W3104657626","https://openalex.org/W3156054180","https://openalex.org/W4206765718"],"related_works":[],"abstract_inverted_index":{"In":[0,102],"any":[1],"ranking":[2,142,157],"system,":[3],"the":[4,36,51,54,64,119,123],"retrieval":[5,29,47,81,115,155],"model":[6],"outputs":[7],"a":[8,12,24,46,57,74,77,127,141,154,160],"single":[9,58],"score":[10,52,125],"for":[11,84,99,114],"document":[13,75,92],"based":[14,143],"on":[15,18,182],"its":[16,70,167],"belief":[17,49,121,139],"how":[19],"relevant":[20],"it":[21],"is":[22,76,178],"to":[23,33,69],"given":[25],"search":[26],"query.":[27],"While":[28],"models":[30,89,116],"have":[31,44],"continued":[32],"improve":[34],"with":[35,67],"introduction":[37],"of":[38,56,73,80,87],"increasingly":[39],"complex":[40],"architectures,":[41],"few":[42],"works":[43],"investigated":[45],"model's":[48,65,120,156],"in":[50,122],"beyond":[53],"scope":[55],"value.":[59],"We":[60,136],"argue":[61],"that":[62,82,147,173],"capturing":[63],"uncertainty":[66,176],"respect":[68],"own":[71],"scoring":[72],"critical":[78],"aspect":[79],"allows":[83],"greater":[85],"use":[86],"current":[88],"across":[90],"new":[91],"distributions,":[93],"collections,":[94],"or":[95],"even":[96],"improving":[97],"effectiveness":[98,158],"down-stream":[100,183],"tasks.":[101],"this":[103,107,138,174],"paper,":[104],"we":[105,171],"address":[106],"problem":[108],"via":[109,140,186],"an":[110],"efficient":[111],"Bayesian":[112,150],"framework":[113,151],"which":[117],"captures":[118],"relevance":[124],"through":[126,159],"stochastic":[128],"process":[129],"while":[130],"adding":[131],"only":[132],"negligible":[133],"computational":[134],"overhead.":[135],"evaluate":[137],"calibration":[144],"metric":[145],"showing":[146],"our":[148],"approximate":[149],"significantly":[152],"improves":[153],"risk":[161],"aware":[162],"reranking":[163],"as":[164,166],"well":[165],"confidence":[168],"calibration.":[169],"Lastly,":[170],"demonstrate":[172],"additional":[175],"information":[177],"actionable":[179],"and":[180],"reliable":[181],"tasks":[184],"represented":[185],"cutoff":[187],"prediction.":[188]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":3},{"year":2024,"cited_by_count":5},{"year":2023,"cited_by_count":7},{"year":2022,"cited_by_count":1},{"year":2021,"cited_by_count":1}],"updated_date":"2026-07-16T13:24:37.021932","created_date":"2021-04-26T00:00:00"}