{"id":"https://openalex.org/W2036239939","doi":"https://doi.org/10.1145/2065003.2065020","title":"Optimizing the cost of information retrieval testcollections","display_name":"Optimizing the cost of information retrieval testcollections","publication_year":2011,"publication_date":"2011-10-28","ids":{"openalex":"https://openalex.org/W2036239939","doi":"https://doi.org/10.1145/2065003.2065020","mag":"2036239939"},"language":"en","primary_location":{"id":"doi:10.1145/2065003.2065020","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2065003.2065020","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 4th workshop on Workshop for Ph.D. students in information & knowledge management","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5070094513","display_name":"Mehdi Hosseini","orcid":"https://orcid.org/0000-0002-6504-4704"},"institutions":[{"id":"https://openalex.org/I45129253","display_name":"University College London","ror":"https://ror.org/02jx3x895","country_code":"GB","type":"education","lineage":["https://openalex.org/I124357947","https://openalex.org/I45129253"]}],"countries":["GB"],"is_corresponding":true,"raw_author_name":"Mehdi Hosseini","raw_affiliation_strings":["University College London, London, United Kingdom"],"affiliations":[{"raw_affiliation_string":"University College London, London, United Kingdom","institution_ids":["https://openalex.org/I45129253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5047286157","display_name":"Ingemar J. Cox","orcid":"https://orcid.org/0000-0002-6662-417X"},"institutions":[{"id":"https://openalex.org/I45129253","display_name":"University College London","ror":"https://ror.org/02jx3x895","country_code":"GB","type":"education","lineage":["https://openalex.org/I124357947","https://openalex.org/I45129253"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Ingemar Cox","raw_affiliation_strings":["University College London, London, United Kingdom"],"affiliations":[{"raw_affiliation_string":"University College London, London, United Kingdom","institution_ids":["https://openalex.org/I45129253"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5113653594","display_name":"Nata\u0161a Mili\u0107-Frayling","orcid":null},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]},{"id":"https://openalex.org/I4210164937","display_name":"Microsoft Research (United Kingdom)","ror":"https://ror.org/05k87vq12","country_code":"GB","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210164937"]}],"countries":["GB","US"],"is_corresponding":false,"raw_author_name":"Natasa Milic-Frayling","raw_affiliation_strings":["Microsoft Research Cambridge, Cambridge, United Kingdom","Microsoft Research Cambridge, Cambridge, United Kingdom#TAB#"],"affiliations":[{"raw_affiliation_string":"Microsoft Research Cambridge, Cambridge, United Kingdom","institution_ids":["https://openalex.org/I4210164937"]},{"raw_affiliation_string":"Microsoft Research Cambridge, Cambridge, United Kingdom#TAB#","institution_ids":["https://openalex.org/I1290206253"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5070094513"],"corresponding_institution_ids":["https://openalex.org/I45129253"],"apc_list":null,"apc_paid":null,"fwci":0.4276,"has_fulltext":false,"cited_by_count":4,"citation_normalized_percentile":{"value":0.71763409,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":94},"biblio":{"volume":null,"issue":null,"first_page":"79","last_page":"82"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.9980000257492065,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.9980000257492065,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.996999979019165,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11269","display_name":"Algorithms and Data Compression","score":0.9835000038146973,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/relevance","display_name":"Relevance (law)","score":0.8675810098648071},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.809403121471405},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7852965593338013},{"id":"https://openalex.org/keywords/construct","display_name":"Construct (python library)","score":0.696395218372345},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.6914100646972656},{"id":"https://openalex.org/keywords/nist","display_name":"NIST","score":0.6738824248313904},{"id":"https://openalex.org/keywords/test","display_name":"Test (biology)","score":0.5773266553878784},{"id":"https://openalex.org/keywords/rank","display_name":"Rank (graph theory)","score":0.5512113571166992},{"id":"https://openalex.org/keywords/ranking","display_name":"Ranking (information retrieval)","score":0.4661226272583008},{"id":"https://openalex.org/keywords/document-retrieval","display_name":"Document retrieval","score":0.41716668009757996},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.39498579502105713},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.12845465540885925},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.08173567056655884}],"concepts":[{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.8675810098648071},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.809403121471405},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7852965593338013},{"id":"https://openalex.org/C2780801425","wikidata":"https://www.wikidata.org/wiki/Q5164392","display_name":"Construct (python library)","level":2,"score":0.696395218372345},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.6914100646972656},{"id":"https://openalex.org/C111219384","wikidata":"https://www.wikidata.org/wiki/Q6954384","display_name":"NIST","level":2,"score":0.6738824248313904},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.5773266553878784},{"id":"https://openalex.org/C164226766","wikidata":"https://www.wikidata.org/wiki/Q7293202","display_name":"Rank (graph theory)","level":2,"score":0.5512113571166992},{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.4661226272583008},{"id":"https://openalex.org/C161156560","wikidata":"https://www.wikidata.org/wiki/Q1638872","display_name":"Document retrieval","level":2,"score":0.41716668009757996},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.39498579502105713},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.12845465540885925},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.08173567056655884},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C114614502","wikidata":"https://www.wikidata.org/wiki/Q76592","display_name":"Combinatorics","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/2065003.2065020","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2065003.2065020","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 4th workshop on Workshop for Ph.D. students in information & knowledge management","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/17","display_name":"Partnerships for the goals","score":0.41999998688697815}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":12,"referenced_works":["https://openalex.org/W193079019","https://openalex.org/W1574528120","https://openalex.org/W1967879792","https://openalex.org/W1976076261","https://openalex.org/W1990170776","https://openalex.org/W2000151369","https://openalex.org/W2075893676","https://openalex.org/W2084726005","https://openalex.org/W2124108136","https://openalex.org/W2948315098","https://openalex.org/W6634383616","https://openalex.org/W6760044250"],"related_works":["https://openalex.org/W2158491338","https://openalex.org/W2150136235","https://openalex.org/W2053591227","https://openalex.org/W2581240705","https://openalex.org/W2041353081","https://openalex.org/W2568183987","https://openalex.org/W3160516639","https://openalex.org/W2118564381","https://openalex.org/W2203835818","https://openalex.org/W1967370444"],"abstract_inverted_index":{"We":[0,24],"consider":[1],"the":[2,49,122,144,154,187],"problem":[3,65],"of":[4,21,32,37,42,58,86,110,124,146,156,186,196,225],"optimally":[5],"allocating":[6],"limited":[7],"resources":[8],"to":[9,45,54,63,67,81,100,103,134,142,194,198],"construct":[10,199],"relevance":[11,91,96,160,234],"judgements":[12,97],"for":[13,35,89,184],"a":[14,29,39,56,76,84,108,114,150],"test":[15,33,115,201],"collection":[16,116],"that":[17,26,139,180],"facilities":[18],"reliable":[19],"evaluation":[20],"retrieval":[22],"systems.":[23,129],"assume":[25],"there":[27,217],"is":[28,140,192],"large":[30,40,219],"set":[31,109],"queries,":[34],"each":[36,185],"which":[38],"number":[41],"documents":[43,148,221],"need":[44],"be":[46,101,195],"judged":[47],"though":[48],"available":[50],"budget":[51,78],"only":[52,105],"permits":[53],"judge":[55],"subset":[57,85],"them.":[59],"A":[60],"candidate":[61],"solution":[62],"this":[64,173],"has":[66,80,99,133],"deal":[68,136],"with,":[69],"at":[70],"least,":[71],"three":[72,188],"challenges.":[73,189],"(i)":[74],"Given":[75],"fixed":[77],"it":[79,98,132],"efficiently":[82],"select":[83],"query-documents":[87],"pairs":[88],"acquiring":[90],"judgements.":[92,235],"(ii)":[93],"With":[94],"collected":[95],"able":[102],"not":[104],"accurately":[106],"evaluate":[107],"systems":[111],"participating":[112],"in":[113,149],"construction":[117],"but":[118],"also":[119],"reliably":[120],"assess":[121],"performance":[123],"new":[125],"as":[126],"yet":[127],"unseen":[128],"(iii)":[130],"Finally,":[131],"properly":[135],"with":[137,158],"uncertainty":[138],"due":[141],"(a)":[143],"presence":[145,155],"unjudged":[147],"rank":[151],"list,":[152],"(b)":[153],"queries":[157],"no":[159],"judgements,":[161],"and":[162,214,223],"(c)":[163],"errors":[164],"caused":[165],"by":[166,203],"human":[167],"assessors":[168],"when":[169],"labelling":[170],"documents.":[171],"In":[172],"thesis":[174],"we":[175],"propose":[176],"an":[177],"optimisation":[178],"framework":[179],"accommodates":[181],"appropriate":[182],"solutions":[183],"Our":[190],"approach":[191],"aimed":[193],"benefit":[197],"IR":[200],"collections":[202,222],"research":[204],"institutes,":[205],"e.g.":[206,212],"NIST,":[207],"or":[208],"commercial":[209],"search":[210],"engines,":[211],"Google":[213],"Bing,":[215],"where":[216],"are":[218],"scale":[220],"loads":[224],"query":[226],"logs":[227],"however":[228],"economic":[229],"constraints":[230],"prohibit":[231],"gathering":[232],"comprehensive":[233]},"counts_by_year":[{"year":2016,"cited_by_count":1},{"year":2015,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}