{"id":"https://openalex.org/W2129341408","doi":"https://doi.org/10.1109/icdim.2007.4444193","title":"Several methods of ranking retrieval systems with partial relevance judgment","display_name":"Several methods of ranking retrieval systems with partial relevance judgment","publication_year":2007,"publication_date":"2007-01-01","ids":{"openalex":"https://openalex.org/W2129341408","doi":"https://doi.org/10.1109/icdim.2007.4444193","mag":"2129341408"},"language":"en","primary_location":{"id":"doi:10.1109/icdim.2007.4444193","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icdim.2007.4444193","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2007 2nd International Conference on Digital Information Management","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5072246520","display_name":"Shengli Wu","orcid":"https://orcid.org/0000-0003-2008-1736"},"institutions":[{"id":"https://openalex.org/I138801177","display_name":"University of Ulster","ror":"https://ror.org/01yp9g959","country_code":"GB","type":"education","lineage":["https://openalex.org/I138801177"]}],"countries":["GB"],"is_corresponding":true,"raw_author_name":"Shengli Wu","raw_affiliation_strings":["School of computing and mathematics, University of Ulster, UK","Sch. of Comput. & Math., Univ. of Ulster, Coleraine"],"affiliations":[{"raw_affiliation_string":"School of computing and mathematics, University of Ulster, UK","institution_ids":["https://openalex.org/I138801177"]},{"raw_affiliation_string":"Sch. of Comput. & Math., Univ. of Ulster, Coleraine","institution_ids":["https://openalex.org/I138801177"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5022378204","display_name":"Sally McClean","orcid":"https://orcid.org/0000-0002-6871-3504"},"institutions":[{"id":"https://openalex.org/I138801177","display_name":"University of Ulster","ror":"https://ror.org/01yp9g959","country_code":"GB","type":"education","lineage":["https://openalex.org/I138801177"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Sally McClean","raw_affiliation_strings":["School of computing and mathematics, University of Ulster, UK","Sch. of Comput. & Math., Univ. of Ulster, Coleraine"],"affiliations":[{"raw_affiliation_string":"School of computing and mathematics, University of Ulster, UK","institution_ids":["https://openalex.org/I138801177"]},{"raw_affiliation_string":"Sch. of Comput. & Math., Univ. of Ulster, Coleraine","institution_ids":["https://openalex.org/I138801177"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":["https://openalex.org/A5072246520"],"corresponding_institution_ids":["https://openalex.org/I138801177"],"apc_list":null,"apc_paid":null,"fwci":0.7913,"has_fulltext":false,"cited_by_count":4,"citation_normalized_percentile":{"value":0.84445192,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":94},"biblio":{"volume":null,"issue":null,"first_page":"13","last_page":"18"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9986000061035156,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9986000061035156,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13083","display_name":"Advanced Text Analysis Techniques","score":0.9832000136375427,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11550","display_name":"Text and Document Classification Technologies","score":0.9807000160217285,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7470890879631042},{"id":"https://openalex.org/keywords/precision-and-recall","display_name":"Precision and recall","score":0.6898447275161743},{"id":"https://openalex.org/keywords/ranking","display_name":"Ranking (information retrieval)","score":0.6545077562332153},{"id":"https://openalex.org/keywords/relevance","display_name":"Relevance (law)","score":0.65303635597229},{"id":"https://openalex.org/keywords/normalization","display_name":"Normalization (sociology)","score":0.6210214495658875},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.5726578831672668},{"id":"https://openalex.org/keywords/recall","display_name":"Recall","score":0.5151466131210327},{"id":"https://openalex.org/keywords/rank","display_name":"Rank (graph theory)","score":0.5012497901916504},{"id":"https://openalex.org/keywords/learning-to-rank","display_name":"Learning to rank","score":0.4751013517379761},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.3659437894821167},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.32827913761138916},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.19237393140792847},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.06730407476425171}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7470890879631042},{"id":"https://openalex.org/C81669768","wikidata":"https://www.wikidata.org/wiki/Q2359161","display_name":"Precision and recall","level":2,"score":0.6898447275161743},{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.6545077562332153},{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.65303635597229},{"id":"https://openalex.org/C136886441","wikidata":"https://www.wikidata.org/wiki/Q926129","display_name":"Normalization (sociology)","level":2,"score":0.6210214495658875},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.5726578831672668},{"id":"https://openalex.org/C100660578","wikidata":"https://www.wikidata.org/wiki/Q18733","display_name":"Recall","level":2,"score":0.5151466131210327},{"id":"https://openalex.org/C164226766","wikidata":"https://www.wikidata.org/wiki/Q7293202","display_name":"Rank (graph theory)","level":2,"score":0.5012497901916504},{"id":"https://openalex.org/C86037889","wikidata":"https://www.wikidata.org/wiki/Q4330127","display_name":"Learning to rank","level":3,"score":0.4751013517379761},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3659437894821167},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.32827913761138916},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.19237393140792847},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.06730407476425171},{"id":"https://openalex.org/C114614502","wikidata":"https://www.wikidata.org/wiki/Q76592","display_name":"Combinatorics","level":1,"score":0.0},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.0},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.0},{"id":"https://openalex.org/C19165224","wikidata":"https://www.wikidata.org/wiki/Q23404","display_name":"Anthropology","level":1,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/icdim.2007.4444193","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icdim.2007.4444193","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2007 2nd International Conference on Digital Information Management","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.46000000834465027,"display_name":"Peace, Justice and strong institutions","id":"https://metadata.un.org/sdg/16"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":21,"referenced_works":["https://openalex.org/W1507150160","https://openalex.org/W1987690005","https://openalex.org/W2015338694","https://openalex.org/W2045334584","https://openalex.org/W2053757624","https://openalex.org/W2057062109","https://openalex.org/W2068098598","https://openalex.org/W2069870183","https://openalex.org/W2073722401","https://openalex.org/W2075893676","https://openalex.org/W2078468464","https://openalex.org/W2090805977","https://openalex.org/W2092488901","https://openalex.org/W2109244020","https://openalex.org/W2110420103","https://openalex.org/W2120308175","https://openalex.org/W2137250419","https://openalex.org/W2146081744","https://openalex.org/W2160892561","https://openalex.org/W2167191270","https://openalex.org/W4233760227"],"related_works":["https://openalex.org/W3127142483","https://openalex.org/W4385565564","https://openalex.org/W2898073868","https://openalex.org/W2358294942","https://openalex.org/W2138488530","https://openalex.org/W2971071571","https://openalex.org/W2798835721","https://openalex.org/W2387658907","https://openalex.org/W2922169395","https://openalex.org/W25098770"],"abstract_inverted_index":{"Some":[0,132],"measures":[1,40],"such":[2,37,76,134],"as":[3,13,77,135],"mean":[4,88],"average":[5,89,100],"precision":[6,10,21,101],"and":[7,22,80,98,141],"recall":[8,23,91],"level":[9,92],"are":[11,25,87,105,146,150],"considered":[12],"good":[14,38],"system-oriented":[15,39,84],"measures,":[16,85],"because":[17],"they":[18],"concern":[19],"both":[20],"that":[24,110],"two":[26],"important":[27],"aspects":[28],"for":[29,153],"effectiveness":[30],"evaluation":[31,74,155],"of":[32,64,116,129,156],"information":[33],"retrieval":[34,59,73,130],"systems.":[35,131],"However,":[36],"suffer":[41],"from":[42],"some":[43],"shortcomings":[44],"when":[45],"partial":[46,65],"relevance":[47,66],"judgment":[48],"is":[49,69],"used.":[50],"In":[51],"this":[52],"paper,":[53],"we":[54],"discuss":[55],"how":[56],"to":[57,125],"rank":[58,126],"systems":[60],"in":[61,71],"the":[62,121,142,154],"condition":[63],"judgment,":[67],"which":[68,86],"common":[70],"major":[72],"events":[75],"TREC":[78],"conferences":[79],"NTCIR":[81],"workshops.":[82],"Four":[83],"precision,":[90,93],"normalized":[94,99],"discount":[95],"cumulative":[96],"gain,":[97],"over":[102,113],"all":[103],"documents,":[104],"discussed.":[106],"Our":[107],"investigation":[108],"shows":[109],"averaging":[111],"values":[112],"a":[114,127],"set":[115],"queries":[117],"may":[118],"not":[119],"be":[120],"most":[122],"reliable":[123],"approach":[124],"group":[128],"alternatives":[133],"Bar":[136],"da":[137],"count.":[138],"Condorcet":[139],"voting,":[140],"zero-one":[143],"normalization":[144],"method,":[145],"investigated.":[147],"Experimental":[148],"results":[149],"also":[151],"presented":[152],"these":[157],"methods.":[158]},"counts_by_year":[{"year":2022,"cited_by_count":1},{"year":2016,"cited_by_count":1},{"year":2013,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}