{"id":"https://openalex.org/W2004918107","doi":"https://doi.org/10.1007/s10791-008-9059-7","title":"On information retrieval metrics designed for evaluation with incomplete relevance assessments","display_name":"On information retrieval metrics designed for evaluation with incomplete relevance assessments","publication_year":2008,"publication_date":"2008-03-19","ids":{"openalex":"https://openalex.org/W2004918107","doi":"https://doi.org/10.1007/s10791-008-9059-7","mag":"2004918107"},"language":"en","primary_location":{"id":"doi:10.1007/s10791-008-9059-7","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s10791-008-9059-7","pdf_url":"https://link.springer.com/content/pdf/10.1007/s10791-008-9059-7.pdf","source":{"id":"https://openalex.org/S79460864","display_name":"Information Retrieval","issn_l":"1386-4564","issn":["1386-4564","1573-7659"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Information Retrieval","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://link.springer.com/content/pdf/10.1007/s10791-008-9059-7.pdf","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5023595778","display_name":"Tetsuya Sakai","orcid":"https://orcid.org/0000-0002-6720-963X"},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Tetsuya Sakai","raw_affiliation_strings":["NewsWatch, Inc., Tokyo, Japan","NewsWatch, Inc., Tokyo, Japan#TAB#"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"NewsWatch, Inc., Tokyo, Japan","institution_ids":[]},{"raw_affiliation_string":"NewsWatch, Inc., Tokyo, Japan#TAB#","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5045875579","display_name":"Noriko Kando","orcid":"https://orcid.org/0000-0002-2133-0215"},"institutions":[{"id":"https://openalex.org/I184597095","display_name":"National Institute of Informatics","ror":"https://ror.org/04ksd4g47","country_code":"JP","type":"facility","lineage":["https://openalex.org/I1319490839","https://openalex.org/I184597095","https://openalex.org/I4210158934"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Noriko Kando","raw_affiliation_strings":["National Institute of Informatics, Tokyo, Japan","(National Institute of Informatics, Tokyo, Japan)"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National Institute of Informatics, Tokyo, Japan","institution_ids":["https://openalex.org/I184597095"]},{"raw_affiliation_string":"(National Institute of Informatics, Tokyo, Japan)","institution_ids":["https://openalex.org/I184597095"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5023595778"],"corresponding_institution_ids":[],"apc_list":{"value":3290,"currency":"USD","value_usd":3290},"apc_paid":{"value":3290,"currency":"USD","value_usd":3290},"fwci":27.9386,"has_fulltext":true,"cited_by_count":128,"citation_normalized_percentile":{"value":0.99326012,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":97,"max":99},"biblio":{"volume":"11","issue":"5","first_page":"447","last_page":"470"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9988999962806702,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9988999962806702,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10215","display_name":"Semantic Web and Ontologies","score":0.996999979019165,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9922999739646912,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/relevance","display_name":"Relevance (law)","score":0.8146835565567017},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7049176692962646},{"id":"https://openalex.org/keywords/learning-to-rank","display_name":"Learning to rank","score":0.5937953591346741},{"id":"https://openalex.org/keywords/discriminative-model","display_name":"Discriminative model","score":0.5795735120773315},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.5685152411460876},{"id":"https://openalex.org/keywords/robustness","display_name":"Robustness (evolution)","score":0.5413039326667786},{"id":"https://openalex.org/keywords/rank","display_name":"Rank (graph theory)","score":0.44974851608276367},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.43945157527923584},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.33792924880981445},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.20636248588562012},{"id":"https://openalex.org/keywords/ranking","display_name":"Ranking (information retrieval)","score":0.16431233286857605}],"concepts":[{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.8146835565567017},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7049176692962646},{"id":"https://openalex.org/C86037889","wikidata":"https://www.wikidata.org/wiki/Q4330127","display_name":"Learning to rank","level":3,"score":0.5937953591346741},{"id":"https://openalex.org/C97931131","wikidata":"https://www.wikidata.org/wiki/Q5282087","display_name":"Discriminative model","level":2,"score":0.5795735120773315},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.5685152411460876},{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.5413039326667786},{"id":"https://openalex.org/C164226766","wikidata":"https://www.wikidata.org/wiki/Q7293202","display_name":"Rank (graph theory)","level":2,"score":0.44974851608276367},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.43945157527923584},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.33792924880981445},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.20636248588562012},{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.16431233286857605},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C114614502","wikidata":"https://www.wikidata.org/wiki/Q76592","display_name":"Combinatorics","level":1,"score":0.0},{"id":"https://openalex.org/C55493867","wikidata":"https://www.wikidata.org/wiki/Q7094","display_name":"Biochemistry","level":1,"score":0.0},{"id":"https://openalex.org/C104317684","wikidata":"https://www.wikidata.org/wiki/Q7187","display_name":"Gene","level":2,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1007/s10791-008-9059-7","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s10791-008-9059-7","pdf_url":"https://link.springer.com/content/pdf/10.1007/s10791-008-9059-7.pdf","source":{"id":"https://openalex.org/S79460864","display_name":"Information Retrieval","issn_l":"1386-4564","issn":["1386-4564","1573-7659"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Information Retrieval","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1007/s10791-008-9059-7","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s10791-008-9059-7","pdf_url":"https://link.springer.com/content/pdf/10.1007/s10791-008-9059-7.pdf","source":{"id":"https://openalex.org/S79460864","display_name":"Information Retrieval","issn_l":"1386-4564","issn":["1386-4564","1573-7659"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Information Retrieval","raw_type":"journal-article"},"sustainable_development_goals":[{"display_name":"Reduced inequalities","score":0.7400000095367432,"id":"https://metadata.un.org/sdg/10"}],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2004918107.pdf","grobid_xml":"https://content.openalex.org/works/W2004918107.grobid-xml"},"referenced_works_count":40,"referenced_works":["https://openalex.org/W597568337","https://openalex.org/W1507711745","https://openalex.org/W1554394657","https://openalex.org/W1584638542","https://openalex.org/W1624787454","https://openalex.org/W1965690069","https://openalex.org/W1967879792","https://openalex.org/W1968927634","https://openalex.org/W1981888042","https://openalex.org/W1983595289","https://openalex.org/W1987690005","https://openalex.org/W1995945562","https://openalex.org/W2001142465","https://openalex.org/W2002371945","https://openalex.org/W2003885352","https://openalex.org/W2005311861","https://openalex.org/W2007671364","https://openalex.org/W2009954908","https://openalex.org/W2045334584","https://openalex.org/W2053100920","https://openalex.org/W2053528567","https://openalex.org/W2053584122","https://openalex.org/W2057495142","https://openalex.org/W2058896506","https://openalex.org/W2068098598","https://openalex.org/W2069870183","https://openalex.org/W2075893676","https://openalex.org/W2081033951","https://openalex.org/W2094424776","https://openalex.org/W2096623622","https://openalex.org/W2109244020","https://openalex.org/W2110202502","https://openalex.org/W2115389155","https://openalex.org/W2124504084","https://openalex.org/W2141175968","https://openalex.org/W2143331230","https://openalex.org/W2293322442","https://openalex.org/W3106889297","https://openalex.org/W6605428294","https://openalex.org/W6636840833"],"related_works":["https://openalex.org/W2965546495","https://openalex.org/W4389116644","https://openalex.org/W2153315159","https://openalex.org/W3103844505","https://openalex.org/W259157601","https://openalex.org/W4205463238","https://openalex.org/W2772359885","https://openalex.org/W3011471740","https://openalex.org/W2884580467","https://openalex.org/W2572315477"],"abstract_inverted_index":{"Modern":[0],"information":[1],"retrieval":[2],"(IR)":[3],"test":[4,72],"collections":[5,73],"have":[6],"grown":[7],"in":[8],"size,":[9],"but":[10],"the":[11,45,57,85,93,102,119,132,158],"available":[12],"manpower":[13],"for":[14,142],"relevance":[15,34,46,64,104,114,173],"assessments":[16],"has":[17],"more":[18],"or":[19,170],"less":[20],"remained":[21],"constant.":[22],"Hence,":[23],"how":[24],"to":[25,62,106,166,177,189,197],"reliably":[26],"evaluate":[27],"and":[28,79,84,88,150,182,194,196,203,213],"compare":[29],"IR":[30,60,90,108],"systems":[31],"using":[32,66],"incomplete":[33,63,113],"data,":[35],"where":[36],"many":[37],"documents":[38],"exist":[39],"that":[40],"were":[41],"never":[42],"examined":[43],"by":[44,185,192,201,216],"assessors,":[47],"is":[48],"receiving":[49],"a":[50,138,143],"lot":[51],"of":[52,59,70,121,134,146,161,211],"attention.":[53],"This":[54],"article":[55],"compares":[56],"robustness":[58],"metrics":[61,169],"assessments,":[65],"four":[67],"different":[68,168,172],"sets":[69],"graded-relevance":[71],"with":[74,111,137],"submitted":[75],"runs\u2014the":[76],"TREC":[77],"2003":[78],"2004":[80],"robust":[81],"track":[82],"data":[83,91,105,174],"NTCIR-6":[86],"Japanese":[87],"Chinese":[89],"from":[92],"crosslingual":[94],"task.":[95],"Following":[96],"previous":[97],"work,":[98],"we":[99,129],"artificially":[100],"reduce":[101],"original":[103],"simulate":[107],"evaluation":[109],"environments":[110],"extremely":[112],"data.":[115],"We":[116,205],"then":[117],"investigate":[118],"effect":[120],"this":[122],"reduction":[123],"on":[124,151],"discriminative":[125],"power":[126],",":[127,155],"which":[128,156],"define":[130],"as":[131],"proportion":[133],"system":[135,163],"pairs":[136],"statistically":[139],"significant":[140],"difference":[141],"given":[144],"probability":[145],"Type":[147],"I":[148],"Error,":[149],"Kendall\u2019s":[152],"rank":[153],"correlation":[154],"reflects":[157],"overall":[159],"resemblance":[160],"two":[162,167,171],"rankings":[164],"according":[165],"sets.":[175],"According":[176],"these":[178],"experiments,":[179],"Q\u2032,":[180],"nDCG\u2032":[181],"AP\u2032":[183],"proposed":[184,191,200],"Sakai":[186],"are":[187],"superior":[188],"bpref":[190,212],"Buckley":[193],"Voorhees":[195],"Rank-Biased":[198,214],"Precision":[199,215],"Moffat":[202],"Zobel.":[204],"also":[206],"point":[207],"out":[208],"some":[209],"weaknesses":[210],"examining":[217],"their":[218],"formal":[219],"definitions.":[220]},"counts_by_year":[{"year":2026,"cited_by_count":4},{"year":2025,"cited_by_count":4},{"year":2024,"cited_by_count":4},{"year":2023,"cited_by_count":4},{"year":2022,"cited_by_count":9},{"year":2021,"cited_by_count":12},{"year":2020,"cited_by_count":11},{"year":2019,"cited_by_count":5},{"year":2018,"cited_by_count":6},{"year":2017,"cited_by_count":8},{"year":2016,"cited_by_count":9},{"year":2015,"cited_by_count":10},{"year":2014,"cited_by_count":7},{"year":2013,"cited_by_count":6},{"year":2012,"cited_by_count":8}],"updated_date":"2026-07-23T08:03:31.855105","created_date":"2025-10-10T00:00:00"}