{"id":"https://openalex.org/W4412673531","doi":"https://doi.org/10.1145/3731120.3744619","title":"Normalized Set-Level Ideal DCG: A More Reliable Early-Stage Retrieval Metric","display_name":"Normalized Set-Level Ideal DCG: A More Reliable Early-Stage Retrieval Metric","publication_year":2025,"publication_date":"2025-07-18","ids":{"openalex":"https://openalex.org/W4412673531","doi":"https://doi.org/10.1145/3731120.3744619"},"language":"en","primary_location":{"id":"doi:10.1145/3731120.3744619","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3731120.3744619","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2025 International ACM SIGIR Conference on Innovative Concepts and Theories in Information Retrieval (ICTIR)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5083382944","display_name":"Xiao Fu","orcid":"https://orcid.org/0000-0003-4676-8608"},"institutions":[{"id":"https://openalex.org/I45129253","display_name":"University College London","ror":"https://ror.org/02jx3x895","country_code":"GB","type":"education","lineage":["https://openalex.org/I124357947","https://openalex.org/I45129253"]}],"countries":["GB"],"is_corresponding":true,"raw_author_name":"Xiao Fu","raw_affiliation_strings":["University College London, London, United Kingdom"],"affiliations":[{"raw_affiliation_string":"University College London, London, United Kingdom","institution_ids":["https://openalex.org/I45129253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5115000447","display_name":"Navdeep Singh Bedi","orcid":"https://orcid.org/0009-0007-5559-2276"},"institutions":[{"id":"https://openalex.org/I57201433","display_name":"Universit\u00e0 della Svizzera italiana","ror":"https://ror.org/03c4atk17","country_code":"CH","type":"education","lineage":["https://openalex.org/I57201433"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Navdeep Singh Bedi","raw_affiliation_strings":["Universita della Svizzera Italiana, Lugano, Switzerland"],"affiliations":[{"raw_affiliation_string":"Universita della Svizzera Italiana, Lugano, Switzerland","institution_ids":["https://openalex.org/I57201433"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5012367814","display_name":"F\u00e1bio Crestani","orcid":"https://orcid.org/0000-0001-8672-0700"},"institutions":[{"id":"https://openalex.org/I57201433","display_name":"Universit\u00e0 della Svizzera italiana","ror":"https://ror.org/03c4atk17","country_code":"CH","type":"education","lineage":["https://openalex.org/I57201433"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Fabio Crestani","raw_affiliation_strings":["Universita della Svizzera Italiana, Lugano, Switzerland"],"affiliations":[{"raw_affiliation_string":"Universita della Svizzera Italiana, Lugano, Switzerland","institution_ids":["https://openalex.org/I57201433"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5058519912","display_name":"Aldo Lipani","orcid":"https://orcid.org/0000-0002-3643-6493"},"institutions":[{"id":"https://openalex.org/I45129253","display_name":"University College London","ror":"https://ror.org/02jx3x895","country_code":"GB","type":"education","lineage":["https://openalex.org/I124357947","https://openalex.org/I45129253"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Aldo Lipani","raw_affiliation_strings":["University College London, London, United Kingdom"],"affiliations":[{"raw_affiliation_string":"University College London, London, United Kingdom","institution_ids":["https://openalex.org/I45129253"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5083382944"],"corresponding_institution_ids":["https://openalex.org/I45129253"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.10015794,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"488","last_page":"496"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9944999814033508,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9944999814033508,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9842000007629395,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10824","display_name":"Image Retrieval and Classification Techniques","score":0.9778000116348267,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/metric","display_name":"Metric (unit)","score":0.6664921045303345},{"id":"https://openalex.org/keywords/stage","display_name":"Stage (stratigraphy)","score":0.6112627983093262},{"id":"https://openalex.org/keywords/ideal","display_name":"Ideal (ethics)","score":0.5877083539962769},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.5603861212730408},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5350568890571594},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.40250474214553833},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.3624710142612457},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.35018959641456604},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.3401700258255005},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.0775877833366394},{"id":"https://openalex.org/keywords/operations-management","display_name":"Operations management","score":0.0738632082939148},{"id":"https://openalex.org/keywords/geology","display_name":"Geology","score":0.06968256831169128}],"concepts":[{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.6664921045303345},{"id":"https://openalex.org/C146357865","wikidata":"https://www.wikidata.org/wiki/Q1123245","display_name":"Stage (stratigraphy)","level":2,"score":0.6112627983093262},{"id":"https://openalex.org/C2776639384","wikidata":"https://www.wikidata.org/wiki/Q840396","display_name":"Ideal (ethics)","level":2,"score":0.5877083539962769},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.5603861212730408},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5350568890571594},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.40250474214553833},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.3624710142612457},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.35018959641456604},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.3401700258255005},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.0775877833366394},{"id":"https://openalex.org/C21547014","wikidata":"https://www.wikidata.org/wiki/Q1423657","display_name":"Operations management","level":1,"score":0.0738632082939148},{"id":"https://openalex.org/C127313418","wikidata":"https://www.wikidata.org/wiki/Q1069","display_name":"Geology","level":0,"score":0.06968256831169128},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3731120.3744619","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3731120.3744619","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2025 International ACM SIGIR Conference on Innovative Concepts and Theories in Information Retrieval (ICTIR)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":12,"referenced_works":["https://openalex.org/W2000431947","https://openalex.org/W2021581601","https://openalex.org/W2037140704","https://openalex.org/W2069870183","https://openalex.org/W2091379987","https://openalex.org/W2096937925","https://openalex.org/W3101622805","https://openalex.org/W3155375847","https://openalex.org/W3169830521","https://openalex.org/W4213009331","https://openalex.org/W4399597806","https://openalex.org/W4405144034"],"related_works":["https://openalex.org/W2590472383","https://openalex.org/W2329386257","https://openalex.org/W2383873009","https://openalex.org/W2503350049","https://openalex.org/W2393580911","https://openalex.org/W2212184297","https://openalex.org/W3156456913","https://openalex.org/W2390945180","https://openalex.org/W2397616145","https://openalex.org/W2394052932"],"abstract_inverted_index":{"Multi-stage":[0],"information":[1],"retrieval":[2,15,80,129,147],"systems":[3],"often":[4],"employ":[5],"a":[6,40,57,75,109,124],"retrieve-then-rerank":[7],"pipeline,":[8],"where":[9],"the":[10,13,31,63,68,87,90,113,150],"quality":[11,81],"of":[12,34,79,92],"initial":[14],"stage":[16,128],"critically":[17],"impacts":[18],"final":[19,114],"ranking":[20,45],"performance.":[21],"Traditional":[22],"set-based":[23],"metrics,":[24,121],"such":[25],"as":[26,139],"recall,":[27],"do":[28],"not":[29],"capture":[30],"nuanced":[32],"impact":[33],"document":[35],"relevance":[36],"distributions,":[37],"leading":[38],"to":[39,118],"potential":[41],"misalignment":[42],"with":[43,112],"downstream":[44],"effectiveness.":[46],"In":[47],"this":[48],"paper,":[49],"we":[50,104],"propose":[51],"Normalized":[52],"Set-Level":[53],"Ideal":[54,65,70],"DCG":[55,66],"(nSIDCG),":[56],"novel,":[58],"optimistic":[59],"metric":[60],"that":[61,106,135],"normalizes":[62],"set-level":[64],"against":[67],"ground-truth":[69],"DCG.":[71],"This":[72],"approach":[73],"ensures":[74],"more":[76,125],"holistic":[77],"evaluation":[78],"by":[82],"taking":[83],"into":[84],"account":[85],"both":[86],"presence":[88],"and":[89],"importance":[91],"relevant":[93],"documents.":[94],"Through":[95],"extensive":[96],"experiments":[97],"on":[98],"TREC":[99],"Deep":[100],"Learning":[101],"Tracks":[102],"(2019-2023),":[103],"demonstrate":[105],"nSIDCG":[107,136],"exhibits":[108],"stronger":[110],"correlation":[111],"nDCG":[115],"scores":[116],"compared":[117],"conventional":[119],"recall-based":[120],"making":[122],"it":[123],"reliable":[126],"early":[127],"assessment":[130],"tool.":[131],"Our":[132],"findings":[133],"suggest":[134],"can":[137],"serve":[138],"an":[140],"effective":[141],"metric,":[142],"helping":[143],"system":[144],"designers":[145],"optimize":[146],"models":[148],"before":[149],"computationally":[151],"expensive":[152],"reranking":[153],"phase.":[154],"Codes:":[155],"https://github.com/RichardFu123/nsidcg.":[156]},"counts_by_year":[],"updated_date":"2025-12-28T23:10:05.387466","created_date":"2025-10-10T00:00:00"}