{"id":"https://openalex.org/W1977437814","doi":"https://doi.org/10.1145/2806416.2806606","title":"Pooled Evaluation Over Query Variations","display_name":"Pooled Evaluation Over Query Variations","alternate_titles":[],"publication_year":2015,"publication_date":"2015-10-17","ids":{"openalex":"https://openalex.org/W1977437814","doi":"https://doi.org/10.1145/2806416.2806606","mag":"1977437814"},"language":"en","primary_location":{"id":"doi:10.1145/2806416.2806606","doi":"https://doi.org/10.1145/2806416.2806606","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2806416.2806606","pdf_url":null,"source":{"id":"https://openalex.org/S4393918199","display_name":"ACM International Conference on Information and Knowledge Management (CIKM)","issn_l":"2155-0751","issn":["2155-0751"],"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 24th ACM International on Conference on Information and Knowledge Management","raw_type":"proceedings-article"},"parent_works":[],"lineage":["https://openalex.org/W1977437814"],"child_works_count":0,"type":"conference-paper","is_traditional":true,"indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://hdl.handle.net/11343/58273","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5081861848","display_name":"Alistair Moffat","orcid":"https://orcid.org/0000-0002-6638-0232","observed_orcids":["https://orcid.org/0000-0002-6638-0232"]},"institutions":[{"id":"https://openalex.org/I165779595","display_name":"The University of Melbourne","ror":"https://ror.org/01ej9dk98","country_code":"AU","type":"education","lineage":["https://openalex.org/I165779595"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Alistair Moffat","raw_affiliation_strings":["The University of Melbourne, Melbourne, Australia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The University of Melbourne, Melbourne, Australia","institution_ids":["https://openalex.org/I165779595"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5085925326","display_name":"Falk Scholer","orcid":"https://orcid.org/0000-0001-9094-0810","observed_orcids":["https://orcid.org/0000-0001-9094-0810"]},"institutions":[{"id":"https://openalex.org/I82951845","display_name":"RMIT University","ror":"https://ror.org/04ttjf776","country_code":"AU","type":"education","lineage":["https://openalex.org/I82951845"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Falk Scholer","raw_affiliation_strings":["RMIT University, Melbourne, Australia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"RMIT University, Melbourne, Australia","institution_ids":["https://openalex.org/I82951845"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5046377370","display_name":"Paul Thomas","orcid":"https://orcid.org/0000-0003-2425-3136","observed_orcids":["https://orcid.org/0000-0003-2425-3136"]},"institutions":[{"id":"https://openalex.org/I1292875679","display_name":"Commonwealth Scientific and Industrial Research Organisation","ror":"https://ror.org/03qn8fb07","country_code":"AU","type":"government","lineage":["https://openalex.org/I1292875679","https://openalex.org/I2801453606","https://openalex.org/I4387156119"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Paul Thomas","raw_affiliation_strings":["CSIRO, Canberra, Australia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"CSIRO, Canberra, Australia","institution_ids":["https://openalex.org/I1292875679"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5017503948","display_name":"Peter Bailey","orcid":"https://orcid.org/0009-0005-9456-1865","observed_orcids":["https://orcid.org/0009-0005-9456-1865"]},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Peter Bailey","raw_affiliation_strings":["Microsoft, Canberra, Australia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft, Canberra, Australia","institution_ids":["https://openalex.org/I1290206253"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":2.0784,"has_fulltext":true,"cited_by_count":35,"citation_normalized_percentile":{"value":0.8779044,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":92,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"1759","last_page":"1762"},"is_retracted":false,"is_pulled":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10203","display_name":"Recommender Systems and Techniques","score":0.0,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/pooling","display_name":"pooling","score":0.6050000190734863},{"id":"https://openalex.org/keywords/information-retrieval-evaluation","display_name":"information retrieval evaluation","score":0.7879999876022339},{"id":"https://openalex.org/keywords/test-collections","display_name":"test collections","score":0.8429999947547913},{"id":"https://openalex.org/keywords/relevance-assessment","display_name":"relevance assessment","score":0.671999990940094}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8323265314102173},{"id":"https://openalex.org/C70437156","wikidata":"https://www.wikidata.org/wiki/Q7228652","display_name":"Pooling","level":2,"score":0.8125929832458496},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.7991410493850708},{"id":"https://openalex.org/C2778334786","wikidata":"https://www.wikidata.org/wiki/Q1586270","display_name":"Variation (astronomy)","level":2,"score":0.6757864952087402},{"id":"https://openalex.org/C99016210","wikidata":"https://www.wikidata.org/wiki/Q5488129","display_name":"Query expansion","level":2,"score":0.6335169672966003},{"id":"https://openalex.org/C118689300","wikidata":"https://www.wikidata.org/wiki/Q7978614","display_name":"Web query classification","level":4,"score":0.6309394240379333},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.5397423505783081},{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.5233235359191895},{"id":"https://openalex.org/C151552104","wikidata":"https://www.wikidata.org/wiki/Q7705809","display_name":"Test suite","level":4,"score":0.4863937199115753},{"id":"https://openalex.org/C79581498","wikidata":"https://www.wikidata.org/wiki/Q1367530","display_name":"Suite","level":2,"score":0.46437492966651917},{"id":"https://openalex.org/C164120249","wikidata":"https://www.wikidata.org/wiki/Q995982","display_name":"Web search query","level":3,"score":0.4638296365737915},{"id":"https://openalex.org/C192028432","wikidata":"https://www.wikidata.org/wiki/Q845739","display_name":"Query language","level":2,"score":0.4499189257621765},{"id":"https://openalex.org/C157692150","wikidata":"https://www.wikidata.org/wiki/Q2919848","display_name":"Query optimization","level":2,"score":0.42750197649002075},{"id":"https://openalex.org/C97854310","wikidata":"https://www.wikidata.org/wiki/Q19541","display_name":"Search engine","level":2,"score":0.2946456968784332},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.18097564578056335},{"id":"https://openalex.org/C128942645","wikidata":"https://www.wikidata.org/wiki/Q1568346","display_name":"Test case","level":3,"score":0.15402984619140625},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.06661581993103027},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C44870925","wikidata":"https://www.wikidata.org/wiki/Q37547","display_name":"Astrophysics","level":1,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C152877465","wikidata":"https://www.wikidata.org/wiki/Q208042","display_name":"Regression analysis","level":2,"score":0.0},{"id":"https://openalex.org/C95457728","wikidata":"https://www.wikidata.org/wiki/Q309","display_name":"History","level":0,"score":0.0},{"id":"https://openalex.org/C166957645","wikidata":"https://www.wikidata.org/wiki/Q23498","display_name":"Archaeology","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/2806416.2806606","doi":"https://doi.org/10.1145/2806416.2806606","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2806416.2806606","pdf_url":null,"source":{"id":"https://openalex.org/S4393918199","display_name":"ACM International Conference on Information and Knowledge Management (CIKM)","issn_l":"2155-0751","issn":["2155-0751"],"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 24th ACM International on Conference on Information and Knowledge Management","raw_type":"proceedings-article"},{"id":"pmh:oai:jupiter.its.unimelb.edu.au:11343/58273","doi":null,"is_oa":true,"landing_page_url":"https://hdl.handle.net/11343/58273","pdf_url":null,"source":{"id":"https://openalex.org/S4377196259","display_name":"Minerva Access (University of Melbourne)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I165779595","host_organization_name":"The University of Melbourne","host_organization_lineage":["https://openalex.org/I165779595"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"24th ACM International on Conference on Information and Knowledge Management (CIKM)","raw_type":"Conference Paper"}],"best_oa_location":{"id":"pmh:oai:jupiter.its.unimelb.edu.au:11343/58273","doi":null,"is_oa":true,"landing_page_url":"https://hdl.handle.net/11343/58273","pdf_url":null,"source":{"id":"https://openalex.org/S4377196259","display_name":"Minerva Access (University of Melbourne)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I165779595","host_organization_name":"The University of Melbourne","host_organization_lineage":["https://openalex.org/I165779595"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"24th ACM International on Conference on Information and Knowledge Management (CIKM)","raw_type":"Conference Paper"},"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G2114630832","display_name":"Effective summaries for search results","funder_award_id":"DP140102655","funder_id":"https://openalex.org/F4320334704","funder_display_name":"Australian Research Council"},{"id":"https://openalex.org/G3023164917","display_name":"Principles, practice, and pragmatics of measurement in experimental computer science","funder_award_id":"DP110101934","funder_id":"https://openalex.org/F4320334704","funder_display_name":"Australian Research Council"}],"funders":[{"id":"https://openalex.org/F4320320386","display_name":"Commonwealth Scientific and Industrial Research Organisation","ror":"https://ror.org/03qn8fb07"},{"id":"https://openalex.org/F4320334704","display_name":"Australian Research Council","ror":"https://ror.org/05mmh0f86"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W1977437814.pdf","grobid_xml":"https://content.openalex.org/works/W1977437814.grobid-xml"},"referenced_works_count":21,"referenced_works":["https://openalex.org/W60779023","https://openalex.org/W132343450","https://openalex.org/W132882894","https://openalex.org/W1491092528","https://openalex.org/W1507150160","https://openalex.org/W2021581601","https://openalex.org/W2053100920","https://openalex.org/W2070740689","https://openalex.org/W2075893676","https://openalex.org/W2090035194","https://openalex.org/W2102698119","https://openalex.org/W2120308175","https://openalex.org/W2141175968","https://openalex.org/W2153252192","https://openalex.org/W2158890594","https://openalex.org/W6602511761","https://openalex.org/W6605428294","https://openalex.org/W6605815150","https://openalex.org/W6629148713","https://openalex.org/W7075649266","https://openalex.org/W7097643049"],"related_works":["https://openalex.org/W7075649266","https://openalex.org/W2559401578","https://openalex.org/W2057495142","https://openalex.org/W2090035194","https://openalex.org/W2534156178","https://openalex.org/W7144285627","https://openalex.org/W2036448375","https://openalex.org/W2165387257","https://openalex.org/W2388783363","https://openalex.org/W3157456408"],"abstract_inverted_index":{"Evaluation":[0],"of":[1,10,13,21,26,29,67,75,83,110,127,131,138,149],"information":[2],"retrieval":[3,150],"systems":[4],"with":[5],"test":[6,120,164],"collections":[7,121],"makes":[8],"use":[9],"a":[11,16,19,81,98],"suite":[12],"fixed":[14],"resources:":[15],"document":[17,31],"corpus;":[18],"set":[20],"topics;":[22],"and":[23,116,167],"associated":[24],"judgments":[25],"the":[27,55,65,72,76,108,136,139,147],"relevance":[28],"each":[30,33],"to":[32,57,90,94],"topic.":[34],"With":[35],"large":[36],"modern":[37],"collections,":[38],"exhaustive":[39],"judging":[40,140],"is":[41,49,142],"not":[42,104,123],"feasible.":[43],"Therefore":[44],"an":[45],"approach":[46],"called":[47],"pooling":[48,86],"typically":[50],"used":[51],"where,":[52],"for":[53,97],"example,":[54],"documents":[56,69,93],"be":[58,61,95,160,169],"judged":[59,96],"can":[60],"determined":[62],"by":[63,80],"taking":[64],"union":[66],"all":[68],"returned":[70,79],"in":[71,163],"top":[73],"positions":[74],"answer":[77],"lists":[78],"range":[82],"systems.":[84],"Conventionally,":[85],"uses":[87],"system":[88,151],"variations":[89],"provide":[91],"diverse":[92],"topic;":[99],"different":[100],"user":[101,111,132,156],"queries":[102],"are":[103],"considered.":[105],"We":[106,153],"explore":[107],"ramifications":[109],"query":[112,133,157],"variability":[113],"on":[114,135],"pooling,":[115],"demonstrate":[117],"that":[118,155],"conventional":[119],"do":[122],"cover":[124],"this":[125],"source":[126],"variation.":[128,152],"The":[129],"effect":[130,148],"variation":[134,158],"size":[137],"pool":[141],"just":[143],"as":[144,146],"strong":[145],"conclude":[154],"should":[159],"incorporated":[161],"early":[162],"collection":[165],"construction,":[166],"cannot":[168],"considered":[170],"effectively":[171],"post":[172],"hoc.":[173]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W1977437814","counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":5},{"year":2024,"cited_by_count":2},{"year":2023,"cited_by_count":5},{"year":2022,"cited_by_count":4},{"year":2021,"cited_by_count":2},{"year":2020,"cited_by_count":2},{"year":2019,"cited_by_count":2},{"year":2017,"cited_by_count":4},{"year":2016,"cited_by_count":6}],"updated_date":"2026-11-17T12:49:18.490984","created_date":"2016-06-24T00:00:00"}
