{"id":"https://openalex.org/W1977437814","doi":"https://doi.org/10.1145/2806416.2806606","title":"Pooled Evaluation Over Query Variations","display_name":"Pooled Evaluation Over Query Variations","publication_year":2015,"publication_date":"2015-10-17","ids":{"openalex":"https://openalex.org/W1977437814","doi":"https://doi.org/10.1145/2806416.2806606","mag":"1977437814"},"language":"en","primary_location":{"id":"doi:10.1145/2806416.2806606","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2806416.2806606","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 24th ACM International on Conference on Information and Knowledge Management","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"http://hdl.handle.net/11343/58273","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5081861848","display_name":"Alistair Moffat","orcid":"https://orcid.org/0000-0002-6638-0232"},"institutions":[{"id":"https://openalex.org/I165779595","display_name":"University of Melbourne","ror":"https://ror.org/01ej9dk98","country_code":"AU","type":"education","lineage":["https://openalex.org/I165779595"]}],"countries":["AU"],"is_corresponding":true,"raw_author_name":"Alistair Moffat","raw_affiliation_strings":["The University of Melbourne, Melbourne, Australia","The University of Melbourne, Melbourne, Australia;"],"affiliations":[{"raw_affiliation_string":"The University of Melbourne, Melbourne, Australia","institution_ids":["https://openalex.org/I165779595"]},{"raw_affiliation_string":"The University of Melbourne, Melbourne, Australia;","institution_ids":["https://openalex.org/I165779595"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5085925326","display_name":"Falk Scholer","orcid":"https://orcid.org/0000-0001-9094-0810"},"institutions":[{"id":"https://openalex.org/I82951845","display_name":"RMIT University","ror":"https://ror.org/04ttjf776","country_code":"AU","type":"education","lineage":["https://openalex.org/I82951845"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Falk Scholer","raw_affiliation_strings":["RMIT University, Melbourne, Australia"],"affiliations":[{"raw_affiliation_string":"RMIT University, Melbourne, Australia","institution_ids":["https://openalex.org/I82951845"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5046377370","display_name":"Paul Thomas","orcid":"https://orcid.org/0000-0003-2425-3136"},"institutions":[{"id":"https://openalex.org/I1292875679","display_name":"Commonwealth Scientific and Industrial Research Organisation","ror":"https://ror.org/03qn8fb07","country_code":"AU","type":"funder","lineage":["https://openalex.org/I1292875679","https://openalex.org/I2801453606","https://openalex.org/I4387156119"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Paul Thomas","raw_affiliation_strings":["CSIRO, Canberra, Australia","CSIRO , Canberra, Australia"],"affiliations":[{"raw_affiliation_string":"CSIRO, Canberra, Australia","institution_ids":["https://openalex.org/I1292875679"]},{"raw_affiliation_string":"CSIRO , Canberra, Australia","institution_ids":["https://openalex.org/I1292875679"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5017503948","display_name":"Peter Bailey","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Peter Bailey","raw_affiliation_strings":["Microsoft, Canberra, Australia","Microsoft, Canberra , Australia"],"affiliations":[{"raw_affiliation_string":"Microsoft, Canberra, Australia","institution_ids":[]},{"raw_affiliation_string":"Microsoft, Canberra , Australia","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5081861848"],"corresponding_institution_ids":["https://openalex.org/I165779595"],"apc_list":null,"apc_paid":null,"fwci":7.946,"has_fulltext":true,"cited_by_count":28,"citation_normalized_percentile":{"value":0.97131741,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"1759","last_page":"1762"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9991000294685364,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10215","display_name":"Semantic Web and Ontologies","score":0.9965000152587891,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8323265314102173},{"id":"https://openalex.org/keywords/pooling","display_name":"Pooling","score":0.8125929832458496},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.7991410493850708},{"id":"https://openalex.org/keywords/variation","display_name":"Variation (astronomy)","score":0.6757864952087402},{"id":"https://openalex.org/keywords/query-expansion","display_name":"Query expansion","score":0.6335169672966003},{"id":"https://openalex.org/keywords/web-query-classification","display_name":"Web query classification","score":0.6309394240379333},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.5397423505783081},{"id":"https://openalex.org/keywords/relevance","display_name":"Relevance (law)","score":0.5233235359191895},{"id":"https://openalex.org/keywords/test-suite","display_name":"Test suite","score":0.4863937199115753},{"id":"https://openalex.org/keywords/suite","display_name":"Suite","score":0.46437492966651917},{"id":"https://openalex.org/keywords/web-search-query","display_name":"Web search query","score":0.4638296365737915},{"id":"https://openalex.org/keywords/query-language","display_name":"Query language","score":0.4499189257621765},{"id":"https://openalex.org/keywords/query-optimization","display_name":"Query optimization","score":0.42750197649002075},{"id":"https://openalex.org/keywords/search-engine","display_name":"Search engine","score":0.2946456968784332},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.18097564578056335},{"id":"https://openalex.org/keywords/test-case","display_name":"Test case","score":0.15402984619140625},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.06661581993103027}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8323265314102173},{"id":"https://openalex.org/C70437156","wikidata":"https://www.wikidata.org/wiki/Q7228652","display_name":"Pooling","level":2,"score":0.8125929832458496},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.7991410493850708},{"id":"https://openalex.org/C2778334786","wikidata":"https://www.wikidata.org/wiki/Q1586270","display_name":"Variation (astronomy)","level":2,"score":0.6757864952087402},{"id":"https://openalex.org/C99016210","wikidata":"https://www.wikidata.org/wiki/Q5488129","display_name":"Query expansion","level":2,"score":0.6335169672966003},{"id":"https://openalex.org/C118689300","wikidata":"https://www.wikidata.org/wiki/Q7978614","display_name":"Web query classification","level":4,"score":0.6309394240379333},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.5397423505783081},{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.5233235359191895},{"id":"https://openalex.org/C151552104","wikidata":"https://www.wikidata.org/wiki/Q7705809","display_name":"Test suite","level":4,"score":0.4863937199115753},{"id":"https://openalex.org/C79581498","wikidata":"https://www.wikidata.org/wiki/Q1367530","display_name":"Suite","level":2,"score":0.46437492966651917},{"id":"https://openalex.org/C164120249","wikidata":"https://www.wikidata.org/wiki/Q995982","display_name":"Web search query","level":3,"score":0.4638296365737915},{"id":"https://openalex.org/C192028432","wikidata":"https://www.wikidata.org/wiki/Q845739","display_name":"Query language","level":2,"score":0.4499189257621765},{"id":"https://openalex.org/C157692150","wikidata":"https://www.wikidata.org/wiki/Q2919848","display_name":"Query optimization","level":2,"score":0.42750197649002075},{"id":"https://openalex.org/C97854310","wikidata":"https://www.wikidata.org/wiki/Q19541","display_name":"Search engine","level":2,"score":0.2946456968784332},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.18097564578056335},{"id":"https://openalex.org/C128942645","wikidata":"https://www.wikidata.org/wiki/Q1568346","display_name":"Test case","level":3,"score":0.15402984619140625},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.06661581993103027},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C44870925","wikidata":"https://www.wikidata.org/wiki/Q37547","display_name":"Astrophysics","level":1,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C152877465","wikidata":"https://www.wikidata.org/wiki/Q208042","display_name":"Regression analysis","level":2,"score":0.0},{"id":"https://openalex.org/C95457728","wikidata":"https://www.wikidata.org/wiki/Q309","display_name":"History","level":0,"score":0.0},{"id":"https://openalex.org/C166957645","wikidata":"https://www.wikidata.org/wiki/Q23498","display_name":"Archaeology","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/2806416.2806606","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2806416.2806606","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 24th ACM International on Conference on Information and Knowledge Management","raw_type":"proceedings-article"},{"id":"pmh:oai:jupiter.its.unimelb.edu.au:11343/58273","is_oa":true,"landing_page_url":"http://hdl.handle.net/11343/58273","pdf_url":"http://hdl.handle.net/11343/58273","source":{"id":"https://openalex.org/S4377196259","display_name":"Minerva Access (University of Melbourne)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I165779595","host_organization_name":"The University of Melbourne","host_organization_lineage":["https://openalex.org/I165779595"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"24th ACM International on Conference on Information and Knowledge Management (CIKM)","raw_type":"Conference Paper"}],"best_oa_location":{"id":"pmh:oai:jupiter.its.unimelb.edu.au:11343/58273","is_oa":true,"landing_page_url":"http://hdl.handle.net/11343/58273","pdf_url":"http://hdl.handle.net/11343/58273","source":{"id":"https://openalex.org/S4377196259","display_name":"Minerva Access (University of Melbourne)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I165779595","host_organization_name":"The University of Melbourne","host_organization_lineage":["https://openalex.org/I165779595"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"24th ACM International on Conference on Information and Knowledge Management (CIKM)","raw_type":"Conference Paper"},"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320320386","display_name":"Commonwealth Scientific and Industrial Research Organisation","ror":"https://ror.org/03qn8fb07"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W1977437814.pdf","grobid_xml":"https://content.openalex.org/works/W1977437814.grobid-xml"},"referenced_works_count":20,"referenced_works":["https://openalex.org/W60779023","https://openalex.org/W132882894","https://openalex.org/W1491092528","https://openalex.org/W1507150160","https://openalex.org/W1507711745","https://openalex.org/W2021581601","https://openalex.org/W2053100920","https://openalex.org/W2070740689","https://openalex.org/W2075893676","https://openalex.org/W2090035194","https://openalex.org/W2102698119","https://openalex.org/W2120308175","https://openalex.org/W2141175968","https://openalex.org/W2153252192","https://openalex.org/W2158890594","https://openalex.org/W6602511761","https://openalex.org/W6605428294","https://openalex.org/W6605431065","https://openalex.org/W6605815150","https://openalex.org/W6629148713"],"related_works":["https://openalex.org/W2572349046","https://openalex.org/W2096359267","https://openalex.org/W2026738364","https://openalex.org/W2113390685","https://openalex.org/W3197639690","https://openalex.org/W2124814993","https://openalex.org/W8514837","https://openalex.org/W2017989738","https://openalex.org/W1981131819","https://openalex.org/W2186703450"],"abstract_inverted_index":{"Evaluation":[0],"of":[1,10,13,21,26,29,67,75,83,110,127,131,138,149],"information":[2],"retrieval":[3,150],"systems":[4],"with":[5],"test":[6,120,164],"collections":[7,121],"makes":[8],"use":[9],"a":[11,16,19,81,98],"suite":[12],"fixed":[14],"resources:":[15],"document":[17,31],"corpus;":[18],"set":[20],"topics;":[22],"and":[23,116,167],"associated":[24],"judgments":[25],"the":[27,55,65,72,76,108,136,139,147],"relevance":[28],"each":[30,33],"to":[32,57,90,94],"topic.":[34],"With":[35],"large":[36],"modern":[37],"collections,":[38],"exhaustive":[39],"judging":[40,140],"is":[41,49,142],"not":[42,104,123],"feasible.":[43],"Therefore":[44],"an":[45],"approach":[46],"called":[47],"pooling":[48,86],"typically":[50],"used":[51],"where,":[52],"for":[53,97],"example,":[54],"documents":[56,69,93],"be":[58,61,95,160,169],"judged":[59,96],"can":[60],"determined":[62],"by":[63,80],"taking":[64],"union":[66],"all":[68],"returned":[70,79],"in":[71,163],"top":[73],"positions":[74],"answer":[77],"lists":[78],"range":[82],"systems.":[84],"Conventionally,":[85],"uses":[87],"system":[88,151],"variations":[89],"provide":[91],"diverse":[92],"topic;":[99],"different":[100],"user":[101,111,132,156],"queries":[102],"are":[103],"considered.":[105],"We":[106,153],"explore":[107],"ramifications":[109],"query":[112,133,157],"variability":[113],"on":[114,135],"pooling,":[115],"demonstrate":[117],"that":[118,155],"conventional":[119],"do":[122],"cover":[124],"this":[125],"source":[126],"variation.":[128,152],"The":[129],"effect":[130,148],"variation":[134,158],"size":[137],"pool":[141],"just":[143],"as":[144,146],"strong":[145],"conclude":[154],"should":[159],"incorporated":[161],"early":[162],"collection":[165],"construction,":[166],"cannot":[168],"considered":[170],"effectively":[171],"post":[172],"hoc.":[173]},"counts_by_year":[{"year":2025,"cited_by_count":4},{"year":2024,"cited_by_count":2},{"year":2023,"cited_by_count":3},{"year":2022,"cited_by_count":4},{"year":2021,"cited_by_count":2},{"year":2020,"cited_by_count":2},{"year":2019,"cited_by_count":1},{"year":2017,"cited_by_count":4},{"year":2016,"cited_by_count":6}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}