{"id":"https://openalex.org/W2066273153","doi":"https://doi.org/10.1145/2348283.2348403","title":"An uncertainty-aware query selection model for evaluation of IR systems","display_name":"An uncertainty-aware query selection model for evaluation of IR systems","publication_year":2012,"publication_date":"2012-08-12","ids":{"openalex":"https://openalex.org/W2066273153","doi":"https://doi.org/10.1145/2348283.2348403","mag":"2066273153"},"language":"en","primary_location":{"id":"doi:10.1145/2348283.2348403","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2348283.2348403","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 35th international ACM SIGIR conference on Research and development in information retrieval","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5070094513","display_name":"Mehdi Hosseini","orcid":"https://orcid.org/0000-0002-6504-4704"},"institutions":[{"id":"https://openalex.org/I45129253","display_name":"University College London","ror":"https://ror.org/02jx3x895","country_code":"GB","type":"education","lineage":["https://openalex.org/I124357947","https://openalex.org/I45129253"]}],"countries":["GB"],"is_corresponding":true,"raw_author_name":"Mehdi Hosseini","raw_affiliation_strings":["university college london, london, United Kingdom"],"affiliations":[{"raw_affiliation_string":"university college london, london, United Kingdom","institution_ids":["https://openalex.org/I45129253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5047286157","display_name":"Ingemar J. Cox","orcid":"https://orcid.org/0000-0002-6662-417X"},"institutions":[{"id":"https://openalex.org/I45129253","display_name":"University College London","ror":"https://ror.org/02jx3x895","country_code":"GB","type":"education","lineage":["https://openalex.org/I124357947","https://openalex.org/I45129253"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Ingemar J. Cox","raw_affiliation_strings":["university college london, london, United Kingdom"],"affiliations":[{"raw_affiliation_string":"university college london, london, United Kingdom","institution_ids":["https://openalex.org/I45129253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5113653594","display_name":"Nata\u0161a Mili\u0107-Frayling","orcid":null},"institutions":[{"id":"https://openalex.org/I4210164937","display_name":"Microsoft Research (United Kingdom)","ror":"https://ror.org/05k87vq12","country_code":"GB","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210164937"]},{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["GB","US"],"is_corresponding":false,"raw_author_name":"Natasa Milic-Frayling","raw_affiliation_strings":["Microsoft Research, Cambridge, United Kingdom","Microsoft Research, Cambridge, United Kingdom ("],"affiliations":[{"raw_affiliation_string":"Microsoft Research, Cambridge, United Kingdom","institution_ids":["https://openalex.org/I4210164937"]},{"raw_affiliation_string":"Microsoft Research, Cambridge, United Kingdom (","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5088114671","display_name":"Milad Shokouhi","orcid":null},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]},{"id":"https://openalex.org/I4210164937","display_name":"Microsoft Research (United Kingdom)","ror":"https://ror.org/05k87vq12","country_code":"GB","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210164937"]}],"countries":["GB","US"],"is_corresponding":false,"raw_author_name":"Milad Shokouhi","raw_affiliation_strings":["Microsoft Research, Cambridge, United Kingdom","Microsoft Research, Cambridge, United Kingdom ("],"affiliations":[{"raw_affiliation_string":"Microsoft Research, Cambridge, United Kingdom","institution_ids":["https://openalex.org/I4210164937"]},{"raw_affiliation_string":"Microsoft Research, Cambridge, United Kingdom (","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5076265623","display_name":"Emine Y\u0131lmaz","orcid":"https://orcid.org/0000-0002-3434-8932"},"institutions":[{"id":"https://openalex.org/I4210164937","display_name":"Microsoft Research (United Kingdom)","ror":"https://ror.org/05k87vq12","country_code":"GB","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210164937"]},{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["GB","US"],"is_corresponding":false,"raw_author_name":"Emine Yilmaz","raw_affiliation_strings":["Microsoft Research, Cambridge, United Kingdom","Microsoft Research, Cambridge, United Kingdom ("],"affiliations":[{"raw_affiliation_string":"Microsoft Research, Cambridge, United Kingdom","institution_ids":["https://openalex.org/I4210164937"]},{"raw_affiliation_string":"Microsoft Research, Cambridge, United Kingdom (","institution_ids":["https://openalex.org/I1290206253"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5070094513"],"corresponding_institution_ids":["https://openalex.org/I45129253"],"apc_list":null,"apc_paid":null,"fwci":2.9359,"has_fulltext":false,"cited_by_count":22,"citation_normalized_percentile":{"value":0.9247016,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"901","last_page":"910"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9987000226974487,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9987000226974487,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9983999729156494,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12761","display_name":"Data Stream Mining Techniques","score":0.987500011920929,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/ranking","display_name":"Ranking (information retrieval)","score":0.850918173789978},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8075684309005737},{"id":"https://openalex.org/keywords/relevance","display_name":"Relevance (law)","score":0.7937530279159546},{"id":"https://openalex.org/keywords/selection","display_name":"Selection (genetic algorithm)","score":0.7219449877738953},{"id":"https://openalex.org/keywords/query-expansion","display_name":"Query expansion","score":0.6160474419593811},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.6066969037055969},{"id":"https://openalex.org/keywords/query-optimization","display_name":"Query optimization","score":0.5132513046264648},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.4638967514038086},{"id":"https://openalex.org/keywords/search-engine","display_name":"Search engine","score":0.4607885777950287},{"id":"https://openalex.org/keywords/web-search-query","display_name":"Web search query","score":0.4568129777908325},{"id":"https://openalex.org/keywords/web-query-classification","display_name":"Web query classification","score":0.435889333486557},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.30130642652511597}],"concepts":[{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.850918173789978},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8075684309005737},{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.7937530279159546},{"id":"https://openalex.org/C81917197","wikidata":"https://www.wikidata.org/wiki/Q628760","display_name":"Selection (genetic algorithm)","level":2,"score":0.7219449877738953},{"id":"https://openalex.org/C99016210","wikidata":"https://www.wikidata.org/wiki/Q5488129","display_name":"Query expansion","level":2,"score":0.6160474419593811},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.6066969037055969},{"id":"https://openalex.org/C157692150","wikidata":"https://www.wikidata.org/wiki/Q2919848","display_name":"Query optimization","level":2,"score":0.5132513046264648},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.4638967514038086},{"id":"https://openalex.org/C97854310","wikidata":"https://www.wikidata.org/wiki/Q19541","display_name":"Search engine","level":2,"score":0.4607885777950287},{"id":"https://openalex.org/C164120249","wikidata":"https://www.wikidata.org/wiki/Q995982","display_name":"Web search query","level":3,"score":0.4568129777908325},{"id":"https://openalex.org/C118689300","wikidata":"https://www.wikidata.org/wiki/Q7978614","display_name":"Web query classification","level":4,"score":0.435889333486557},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.30130642652511597},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1145/2348283.2348403","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2348283.2348403","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 35th international ACM SIGIR conference on Research and development in information retrieval","raw_type":"proceedings-article"},{"id":"pmh:oai:CiteSeerX.psu:10.1.1.374.1343","is_oa":false,"landing_page_url":"http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.374.1343","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://www0.cs.ucl.ac.uk/staff/ingemar/Content/papers/2012/SIGIR2012.pdf","raw_type":"text"},{"id":"pmh:oai:eprints.ucl.ac.uk.OAI2:1363034","is_oa":false,"landing_page_url":"http://discovery.ucl.ac.uk/1363034/","pdf_url":null,"source":{"id":"https://openalex.org/S4306400024","display_name":"UCL Discovery (University College London)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I45129253","host_organization_name":"University College London","host_organization_lineage":["https://openalex.org/I45129253"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"     In:    (pp. pp. 901-910).   (2012)     ","raw_type":"Proceedings paper"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":24,"referenced_works":["https://openalex.org/W193079019","https://openalex.org/W1549656520","https://openalex.org/W1574528120","https://openalex.org/W1618905105","https://openalex.org/W1938221098","https://openalex.org/W1967879792","https://openalex.org/W1976076261","https://openalex.org/W1983595289","https://openalex.org/W1990170776","https://openalex.org/W1994960885","https://openalex.org/W2009954908","https://openalex.org/W2021856948","https://openalex.org/W2027359109","https://openalex.org/W2056099894","https://openalex.org/W2057028302","https://openalex.org/W2057495142","https://openalex.org/W2062236692","https://openalex.org/W2075893676","https://openalex.org/W2102661342","https://openalex.org/W2119821739","https://openalex.org/W2124504084","https://openalex.org/W2154884126","https://openalex.org/W4298443330","https://openalex.org/W7052102089"],"related_works":["https://openalex.org/W2096359267","https://openalex.org/W2026738364","https://openalex.org/W2572349046","https://openalex.org/W2013069866","https://openalex.org/W8514837","https://openalex.org/W1793997780","https://openalex.org/W2146885082","https://openalex.org/W3125756434","https://openalex.org/W4232633635","https://openalex.org/W3049728138"],"abstract_inverted_index":{"We":[0,105],"propose":[1],"a":[2,9,85,120],"mathematical":[3,24],"framework":[4],"for":[5,11,79],"query":[6,55,81,86],"selection":[7,56,101],"as":[8,60,117,119],"mechanism":[10],"reducing":[12],"the":[13,28,31,39,45,80,89,100,107,110,136,153,158,167],"cost":[14],"of":[15,41,102,109,123,145],"constructing":[16],"information":[17],"retrieval":[18,32],"test":[19,115,121],"collections.":[20],"In":[21],"particular,":[22],"our":[23],"formulation":[25],"explicitly":[26],"models":[27],"uncertainty":[29],"in":[30],"effectiveness":[33,108],"metrics":[34],"that":[35,62,73,135,163],"is":[36,48,87,149],"introduced":[37],"by":[38,139,161],"absence":[40],"relevance":[42,75,91],"judgments.":[43],"Since":[44],"optimization":[46],"problem":[47],"computationally":[49],"intractable,":[50],"we":[51],"devise":[52],"an":[53,64,124],"adaptive":[54],"algorithm,":[57],"referred":[58],"to":[59,98],"Adaptive,":[61],"provides":[63],"approximate":[65],"solution.":[66],"Adaptive":[67,140],"selects":[68],"queries":[69,137,162],"iteratively":[70],"and":[71,95],"assumes":[72],"no":[74],"judgments":[76],"are":[77,93],"available":[78],"under":[82],"consideration.":[83],"Once":[84],"selected,":[88],"associated":[90],"assessments":[92],"acquired":[94],"then":[96],"used":[97],"aid":[99],"subsequent":[103],"queries.":[104,130],"demonstrate":[106],"algorithm":[111],"on":[112],"two":[113],"TREC":[114],"collections":[116],"well":[118],"collection":[122],"online":[125],"search":[126],"engine":[127],"with":[128,152],"1000":[129],"Our":[131],"experimental":[132],"results":[133],"show":[134],"chosen":[138],"produce":[141],"reliable":[142],"performance":[143],"ranking":[144,148,156],"systems.":[146],"The":[147],"better":[150],"correlated":[151],"actual":[154],"systems":[155],"than":[157],"rankings":[159],"produced":[160],"were":[164],"selected":[165],"using":[166],"considered":[168],"baseline":[169],"methods.":[170]},"counts_by_year":[{"year":2023,"cited_by_count":1},{"year":2022,"cited_by_count":2},{"year":2021,"cited_by_count":1},{"year":2020,"cited_by_count":2},{"year":2019,"cited_by_count":5},{"year":2018,"cited_by_count":2},{"year":2017,"cited_by_count":2},{"year":2016,"cited_by_count":3},{"year":2015,"cited_by_count":2},{"year":2014,"cited_by_count":2}],"updated_date":"2026-04-05T17:49:38.594831","created_date":"2025-10-10T00:00:00"}