{"id":"https://openalex.org/W2898078244","doi":"https://doi.org/10.1145/3269206.3271719","title":"Meta-Analysis for Retrieval Experiments Involving Multiple Test Collections","display_name":"Meta-Analysis for Retrieval Experiments Involving Multiple Test Collections","publication_year":2018,"publication_date":"2018-10-17","ids":{"openalex":"https://openalex.org/W2898078244","doi":"https://doi.org/10.1145/3269206.3271719","mag":"2898078244"},"language":"en","primary_location":{"id":"doi:10.1145/3269206.3271719","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3269206.3271719","pdf_url":null,"source":null,"license":"public-domain","license_id":"https://openalex.org/licenses/public-domain","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 27th ACM International Conference on Information and Knowledge Management","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5053747568","display_name":"Ian Soboroff","orcid":"https://orcid.org/0000-0003-2363-3014"},"institutions":[{"id":"https://openalex.org/I1321296531","display_name":"National Institute of Standards and Technology","ror":"https://ror.org/05xpvk416","country_code":"US","type":"funder","lineage":["https://openalex.org/I1321296531","https://openalex.org/I1343035065"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Ian Soboroff","raw_affiliation_strings":["National Institute of Standards and Technology, Gaithersburg, MD, USA"],"affiliations":[{"raw_affiliation_string":"National Institute of Standards and Technology, Gaithersburg, MD, USA","institution_ids":["https://openalex.org/I1321296531"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5053747568"],"corresponding_institution_ids":["https://openalex.org/I1321296531"],"apc_list":null,"apc_paid":null,"fwci":0.3927,"has_fulltext":false,"cited_by_count":10,"citation_normalized_percentile":{"value":0.71953493,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":97},"biblio":{"volume":null,"issue":null,"first_page":"713","last_page":"722"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9987000226974487,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9987000226974487,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9912999868392944,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13083","display_name":"Advanced Text Analysis Techniques","score":0.9905999898910522,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/pooling","display_name":"Pooling","score":0.8401059508323669},{"id":"https://openalex.org/keywords/standardization","display_name":"Standardization","score":0.8307901620864868},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.749416708946228},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.6184267401695251},{"id":"https://openalex.org/keywords/test","display_name":"Test (biology)","score":0.5809056758880615},{"id":"https://openalex.org/keywords/relevance","display_name":"Relevance (law)","score":0.5068004727363586},{"id":"https://openalex.org/keywords/statistical-hypothesis-testing","display_name":"Statistical hypothesis testing","score":0.49051278829574585},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.48904523253440857},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.42591649293899536},{"id":"https://openalex.org/keywords/null-hypothesis","display_name":"Null hypothesis","score":0.41988953948020935},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.3370920717716217},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.31274867057800293},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.21572354435920715},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.10753738880157471}],"concepts":[{"id":"https://openalex.org/C70437156","wikidata":"https://www.wikidata.org/wiki/Q7228652","display_name":"Pooling","level":2,"score":0.8401059508323669},{"id":"https://openalex.org/C188087704","wikidata":"https://www.wikidata.org/wiki/Q369577","display_name":"Standardization","level":2,"score":0.8307901620864868},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.749416708946228},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.6184267401695251},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.5809056758880615},{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.5068004727363586},{"id":"https://openalex.org/C87007009","wikidata":"https://www.wikidata.org/wiki/Q210832","display_name":"Statistical hypothesis testing","level":2,"score":0.49051278829574585},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.48904523253440857},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.42591649293899536},{"id":"https://openalex.org/C191988596","wikidata":"https://www.wikidata.org/wiki/Q628374","display_name":"Null hypothesis","level":2,"score":0.41988953948020935},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.3370920717716217},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.31274867057800293},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.21572354435920715},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.10753738880157471},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3269206.3271719","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3269206.3271719","pdf_url":null,"source":null,"license":"public-domain","license_id":"https://openalex.org/licenses/public-domain","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 27th ACM International Conference on Information and Knowledge Management","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education","score":0.6299999952316284}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":27,"referenced_works":["https://openalex.org/W32360141","https://openalex.org/W71228264","https://openalex.org/W1554394657","https://openalex.org/W1587004086","https://openalex.org/W1976076261","https://openalex.org/W1976535830","https://openalex.org/W2006996936","https://openalex.org/W2013104098","https://openalex.org/W2037140704","https://openalex.org/W2058200372","https://openalex.org/W2058896506","https://openalex.org/W2073722401","https://openalex.org/W2076227143","https://openalex.org/W2076598705","https://openalex.org/W2085601930","https://openalex.org/W2098162425","https://openalex.org/W2111310810","https://openalex.org/W2119085739","https://openalex.org/W2146524295","https://openalex.org/W2161701454","https://openalex.org/W2164547069","https://openalex.org/W2171853361","https://openalex.org/W2410091547","https://openalex.org/W2515650098","https://openalex.org/W2522871920","https://openalex.org/W2790055648","https://openalex.org/W4233760227"],"related_works":["https://openalex.org/W2903799831","https://openalex.org/W2348480869","https://openalex.org/W2485444738","https://openalex.org/W2979681497","https://openalex.org/W3015822731","https://openalex.org/W2585981346","https://openalex.org/W2049531272","https://openalex.org/W3003558595","https://openalex.org/W4242056622","https://openalex.org/W1977503882"],"abstract_inverted_index":{"Traditional":[0],"practice":[1],"recommends":[2],"that":[3,18],"information":[4],"retrieval":[5],"experiments":[6,63,76],"be":[7],"run":[8],"over":[9,129],"multiple":[10,65],"test":[11,45,66],"collections,":[12],"to":[13,24,26,86],"support,":[14],"if":[15],"not":[16,40],"prove,":[17],"gains":[19],"in":[20,97,118,133],"performance":[21],"are":[22,39],"likely":[23],"generalize":[25],"other":[27],"collections":[28],"or":[29],"tasks.":[30],"However,":[31],"because":[32],"of":[33],"the":[34,70,84,98],"pooling":[35],"assumptions,":[36],"evaluation":[37],"scores":[38],"directly":[41],"comparable":[42],"across":[43],"different":[44],"collections.":[46,67],"We":[47,68],"present":[48],"a":[49,56,112,119],"widely-used":[50],"statistical":[51,116],"tool,":[52],"\\em":[53],"meta-analysis,":[54],"as":[55],"framework":[57],"for":[58],"reporting":[59,103],"results":[60,85],"from":[61,89],"IR":[62],"using":[64,130],"demonstrate":[69],"meta-analytical":[71],"approach":[72],"through":[73],"two":[74],"standard":[75],"on":[77,114],"stemming":[78],"and":[79,82,110],"pseudo-relevance":[80],"feedback,":[81],"compare":[83],"those":[87],"obtained":[88],"score":[90,101,108],"standardization.":[91],"Meta-analysis":[92],"incorporates":[93],"several":[94],"recent":[95],"recommendations":[96],"literature,":[99],"including":[100],"standardization,":[102],"effect":[104],"sizes":[105],"rather":[106],"than":[107],"differences,":[109],"avoiding":[111],"reliance":[113],"null-hypothesis":[115],"testing,":[117],"unified":[120],"approach.":[121],"It":[122],"therefore":[123],"represents":[124],"an":[125],"important":[126],"methodological":[127],"improvement":[128],"these":[131],"techniques":[132],"isolation.":[134]},"counts_by_year":[{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":1},{"year":2023,"cited_by_count":4},{"year":2022,"cited_by_count":2},{"year":2021,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}